Upload 3 files

app.py

@@ -0,0 +1,61 @@
+import os
+
+import streamlit as st
+from dotenv import load_dotenv
+from langchain.callbacks.base import BaseCallbackHandler
+from langchain.chains import ConversationalRetrievalChain
+from langchain.chat_models import ChatOpenAI
+from langchain.embeddings import OpenAIEmbeddings
+from langchain.memory import ConversationBufferMemory
+from langchain.memory.chat_message_histories import StreamlitChatMessageHistory
+from langchain.vectorstores import Chroma
+
+load_dotenv()
+
+website_url = os.environ.get('WEBSITE_URL', 'a website')
+
+st.set_page_config(page_title=f'Chat with {website_url}')
+st.title('Chat with a website')
+
+@st.cache_resource(ttl='1h')
+def get_retriever():
+    embeddings = OpenAIEmbeddings()
+    vectordb = Chroma(persist_directory='db', embedding_function=embeddings)
+
+    retriever = vectordb.as_retriever(search_type='mmr')
+
+    return retriever
+
+class StreamHandler(BaseCallbackHandler):
+    def __init__(self, container: st.delta_generator.DeltaGenerator, initial_text: str = ''):
+        self.container = container
+        self.text = initial_text
+
+    def on_llm_new_token(self, token: str, **kwargs) -> None:
+        self.text += token
+        self.container.markdown(self.text)
+
+retriever = get_retriever()
+
+msgs = StreamlitChatMessageHistory()
+memory = ConversationBufferMemory(memory_key='chat_history', chat_memory=msgs, return_messages=True)
+
+llm = ChatOpenAI(model_name='gpt-3.5-turbo', temperature=0, streaming=True)
+qa_chain = ConversationalRetrievalChain.from_llm(
+    llm, retriever=retriever, memory=memory, verbose=False
+)
+
+if st.sidebar.button('Clear message history') or len(msgs.messages) == 0:
+    msgs.clear()
+    msgs.add_ai_message(f'Ask me anything about {website_url}!')
+
+avatars = {'human': 'user', 'ai': 'assistant'}
+for msg in msgs.messages:
+    st.chat_message(avatars[msg.type]).write(msg.content)
+
+if user_query := st.chat_input(placeholder='Ask me anything!'):
+    st.chat_message('user').write(user_query)
+
+    with st.chat_message('assistant'):
+        stream_handler = StreamHandler(st.empty())
+        response = qa_chain.run(user_query, callbacks=[stream_handler])

requirements.txt

@@ -0,0 +1,7 @@
+apify-client
+chromadb
+langchain
+openai
+python-dotenv
+streamlit
+tiktoken

scrape.py

@@ -0,0 +1,40 @@
+import os
+
+from apify_client import ApifyClient
+from dotenv import load_dotenv
+from langchain.document_loaders import ApifyDatasetLoader
+from langchain.document_loaders.base import Document
+from langchain.embeddings.openai import OpenAIEmbeddings
+from langchain.text_splitter import RecursiveCharacterTextSplitter
+from langchain.vectorstores import Chroma
+
+# Load environment variables from a .env file
+load_dotenv()
+
+if __name__ == '__main__':
+    apify_client = ApifyClient(os.environ.get('APIFY_API_TOKEN'))
+    website_url = os.environ.get('WEBSITE_URL')
+    print(f'Extracting data from "{website_url}". Please wait...')
+    actor_run_info = apify_client.actor('apify/website-content-crawler').call(
+        run_input={'startUrls': [{'url': website_url}]}
+    )
+    print('Saving data into the vector database. Please wait...')
+    loader = ApifyDatasetLoader(
+        dataset_id=actor_run_info['defaultDatasetId'],
+        dataset_mapping_function=lambda item: Document(
+            page_content=item['text'] or '', metadata={'source': item['url']}
+        ),
+    )
+    documents = loader.load()
+    text_splitter = RecursiveCharacterTextSplitter(chunk_size=1500, chunk_overlap=100)
+    docs = text_splitter.split_documents(documents)
+
+    embedding = OpenAIEmbeddings()
+
+    vectordb = Chroma.from_documents(
+        documents=docs,
+        embedding=embedding,
+        persist_directory='db2',
+    )
+    vectordb.persist()
+    print('All done!')