Upload folder using huggingface_hub

.idea/workspace.xml

@@ -4,10 +4,7 @@
     <option name="autoReloadType" value="SELECTIVE" />
   </component>
   <component name="ChangeListManager">
-    <list default="true" id="389d532e-4eaf-4367-8bd6-095ab4d8b955" name="Changes" comment="">
-      <change beforePath="$PROJECT_DIR$/.gitignore" beforeDir="false" afterPath="$PROJECT_DIR$/.gitignore" afterDir="false" />
-      <change beforePath="$PROJECT_DIR$/chat_engine.py" beforeDir="false" afterPath="$PROJECT_DIR$/chat_engine.py" afterDir="false" />
-    </list>
+    <list default="true" id="389d532e-4eaf-4367-8bd6-095ab4d8b955" name="Changes" comment="" />
     <option name="SHOW_DIALOG" value="false" />
     <option name="HIGHLIGHT_CONFLICTS" value="true" />
     <option name="HIGHLIGHT_NON_ACTIVE_CHANGELIST" value="false" />
@@ -23,6 +20,31 @@
   <component name="Git.Settings">
     <option name="RECENT_GIT_ROOT_PATH" value="$PROJECT_DIR$" />
   </component>
+  <component name="GitToolBoxStore">
+    <option name="recentBranches">
+      <RecentBranches>
+        <option name="branchesForRepo">
+          <list>
+            <RecentBranchesForRepo>
+              <option name="branches">
+                <list>
+                  <RecentBranch>
+                    <option name="branchName" value="chat-engine" />
+                    <option name="lastUsedInstant" value="1693355057" />
+                  </RecentBranch>
+                  <RecentBranch>
+                    <option name="branchName" value="master" />
+                    <option name="lastUsedInstant" value="1693355056" />
+                  </RecentBranch>
+                </list>
+              </option>
+              <option name="repositoryRootUrl" value="file://$PROJECT_DIR$" />
+            </RecentBranchesForRepo>
+          </list>
+        </option>
+      </RecentBranches>
+    </option>
+  </component>
   <component name="MarkdownSettingsMigration">
     <option name="stateVersion" value="1" />
   </component>
@@ -40,7 +62,7 @@
     "RunOnceActivity.OpenProjectViewOnStart": "true",
     "RunOnceActivity.ShowReadmeOnStart": "true",
     "WebServerToolWindowFactoryState": "false",
-    "git-widget-placeholder": "master",
+    "git-widget-placeholder": "chat-engine",
     "last_opened_file_path": "/home/draco/PycharmProjects/customChatBotDemo",
     "node.js.detected.package.eslint": "true",
     "node.js.detected.package.tslint": "true",
@@ -90,7 +112,7 @@
       <workItem from="1693344352959" duration="1601000" />
       <workItem from="1693345962391" duration="4297000" />
       <workItem from="1693351210434" duration="1643000" />
-      <workItem from="1693353619587" duration="571000" />
+      <workItem from="1693353619587" duration="1897000" />
     </task>
     <servers />
   </component>

main.py

@@ -1,46 +1,37 @@
 import gradio as gr
-import os
-import openai
 from dotenv import load_dotenv
-from langchain.chains import ConversationalRetrievalChain
-from langchain.chat_models import ChatOpenAI
-from langchain.document_loaders import DirectoryLoader
-from langchain.indexes.vectorstore import VectorStoreIndexWrapper
-from langchain.indexes import VectorstoreIndexCreator
-from langchain.llms import OpenAI
-from langchain.embeddings import OpenAIEmbeddings
-from langchain.vectorstores.chroma import Chroma
+from llama_index import (
+    ServiceContext,
+    VectorStoreIndex,
+    SimpleDirectoryReader,
+    set_global_service_context,
+)
+from llama_index.llms import OpenAI
+import openai
+import constants
 
 load_dotenv()
-os.getenv("OPENAI_API_KEY")
+# openai.api_key = os.getenv("OPENAI_API_KEY")
+openai.api_key = constants.APIKEY
 
-PERSIST = False
-
-if PERSIST and os.path.exists("persist"):
-    print("Reusing index...\n")
-    vectorstore = Chroma(persist_directory="persist", embedding_function=OpenAIEmbeddings())
-    index = VectorStoreIndexWrapper(vectorstore=vectorstore)
-    index.load("persist")
-else:
-    loader = DirectoryLoader("data/")
-    if PERSIST:
-        index = VectorstoreIndexCreator(vectorstore_kwargs={"persist_directory": "persist"}).from_loaders([loader])
-    else:
-        index = VectorstoreIndexCreator().from_loaders([loader])
+service_context = ServiceContext.from_defaults(
+    llm=OpenAI(model="gpt-3.5-turbo-16k", temperature=0)
+)
+set_global_service_context(service_context)
 
-use_gpt_4 = True
-model_name = "gpt-4" if use_gpt_4 else "gpt-3.5-turbo-16k"
-chain = ConversationalRetrievalChain.from_llm(llm=ChatOpenAI(model=model_name),
-                                              retriever=index.vectorstore.as_retriever(search_kwargs={"k": 1}))
+data = SimpleDirectoryReader(input_dir="./data/").load_data()
 
-chat_history = []
+index = VectorStoreIndex.from_documents(data)
+chat_engine = index.as_chat_engine(chat_mode="context")
 
 
 def answer_question(question):
-    global chat_history
-    result = chain({"question": question, "chat_history": chat_history})
-    chat_history.append((question, result['answer']))
-    return result['answer']
+    response = chat_engine.stream_chat(question)
+    response_text = ""
+    for token in response.response_gen:
+        print(token, end="")
+        response_text += token
+    return response_text
 
 
 iface = gr.Interface(

requirements.txt

@@ -1,22 +1,4 @@
-langchain==0.0.276
-openai==0.27.9
-pandasai==1.0.9
-pdfplumber
-pypdf==3.15.4
-streamlit==1.26.0
-streamlit-chat==0.1.1
-tiktoken==0.4.0
-youtube-transcript-api==0.6.1
-faiss-cpu
-gradio
-unstructured
-chromadb
-llama-index==0.8.2
-transformers
-span-marker
-nltk
-llama-hub
-tesseract
-pytesseract
-python-dotenv
-PyCryptodome
+gradio~=3.41.2
+openai~=0.27.9
+python-dotenv~=1.0.0
+llama-index==0.8.2

venv/lib/python3.11/site-packages/gradio/launches.json

@@ -1 +1 @@
-{"launches": 23}
+{"launches": 25}