update flake, fix ingest steamlit compatibility bug

backend/generate_metadata.py

@@ -1,4 +1,5 @@
 import os
+import io
 import argparse
 import json
 import openai
@@ -12,13 +13,13 @@ from langchain_text_splitters import RecursiveCharacterTextSplitter
 load_dotenv()
 
 
-def ingest(file_path):
-    extension = file_path.split('.')[-1]
-    ext = extension.lower()
-    if ext == 'pdf':
-        loader = UnstructuredPDFLoader(file_path)
-    elif ext == 'txt':
-        loader = TextLoader(file_path)
+import io
+
+def ingest(file_obj, file_ext='pdf'):
+    if file_ext == 'pdf':
+        loader = UnstructuredPDFLoader(file_obj)
+    elif file_ext == 'txt':
+        loader = TextLoader(file_obj)
     else:
         raise NotImplementedError('Only .txt or .pdf files are supported')
 
@@ -29,7 +30,7 @@ def ingest(file_path):
         "\n\n",
         "\n",
         " ",
-        ",",        
+        ",",
         "\uff0c",  # Fullwidth comma
         "\u3001",  # Ideographic comma
         "\uff0e",  # Fullwidth full stop

flake.nix

@@ -14,6 +14,9 @@
     devShells.${system}.default = pkgs.mkShell {
       packages = [
         (pkgs.python311.withPackages (python-pkgs: [
+          python-pkgs.pip # VsCode starts
+          python-pkgs.jupyter
+          python-pkgs.notebook # VsCode ends
           python-pkgs.numpy
           python-pkgs.pandas
           python-pkgs.scipy
@@ -23,15 +26,20 @@
           python-pkgs.langchain
           python-pkgs.langchain-text-splitters
           python-pkgs.unstructured
+          python-pkgs.wrapt # unstructured[local-inference] starts
+          python-pkgs.iso-639
+          python-pkgs.emoji
+          python-pkgs.pillow-heif
+          python-pkgs.magic
+          python-pkgs.poppler-qt5
+          python-pkgs.pytesseract
+          python-pkgs.langdetect # unstructured[local-inference] ends
           python-pkgs.openai
           python-pkgs.pydantic
           python-pkgs.python-dotenv
           python-pkgs.configargparse
           python-pkgs.streamlit
-          python-pkgs.pip
           python-pkgs.lark
-          python-pkgs.jupyter
-          python-pkgs.notebook
           python-pkgs.sentence-transformers
           pkgs.unstructured-api
         ]))