Upload 6 files

.env.example

@@ -0,0 +1,3 @@
+OPENAI_API_KEY=your_api_key
+APIFY_API_TOKEN=your_api_key
+WEBSITE_URL="https://docs.apify.com/platform"

.gitignore

@@ -0,0 +1,163 @@
+# Byte-compiled / optimized / DLL files
+__pycache__/
+*.py[cod]
+*$py.class
+
+# C extensions
+*.so
+
+# Distribution / packaging
+.Python
+build/
+develop-eggs/
+dist/
+downloads/
+eggs/
+.eggs/
+lib/
+lib64/
+parts/
+sdist/
+var/
+wheels/
+share/python-wheels/
+*.egg-info/
+.installed.cfg
+*.egg
+MANIFEST
+
+# PyInstaller
+#  Usually these files are written by a python script from a template
+#  before PyInstaller builds the exe, so as to inject date/other infos into it.
+*.manifest
+*.spec
+
+# Installer logs
+pip-log.txt
+pip-delete-this-directory.txt
+
+# Unit test / coverage reports
+htmlcov/
+.tox/
+.nox/
+.coverage
+.coverage.*
+.cache
+nosetests.xml
+coverage.xml
+*.cover
+*.py,cover
+.hypothesis/
+.pytest_cache/
+cover/
+
+# Translations
+*.mo
+*.pot
+
+# Django stuff:
+*.log
+local_settings.py
+db.sqlite3
+db.sqlite3-journal
+
+# Flask stuff:
+instance/
+.webassets-cache
+
+# Scrapy stuff:
+.scrapy
+
+# Sphinx documentation
+docs/_build/
+
+# PyBuilder
+.pybuilder/
+target/
+
+# Jupyter Notebook
+.ipynb_checkpoints
+
+# IPython
+profile_default/
+ipython_config.py
+
+# pyenv
+#   For a library or package, you might want to ignore these files since the code is
+#   intended to run in multiple environments; otherwise, check them in:
+# .python-version
+
+# pipenv
+#   According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control.
+#   However, in case of collaboration, if having platform-specific dependencies or dependencies
+#   having no cross-platform support, pipenv may install dependencies that don't work, or not
+#   install all needed dependencies.
+#Pipfile.lock
+
+# poetry
+#   Similar to Pipfile.lock, it is generally recommended to include poetry.lock in version control.
+#   This is especially recommended for binary packages to ensure reproducibility, and is more
+#   commonly ignored for libraries.
+#   https://python-poetry.org/docs/basic-usage/#commit-your-poetrylock-file-to-version-control
+#poetry.lock
+
+# pdm
+#   Similar to Pipfile.lock, it is generally recommended to include pdm.lock in version control.
+#pdm.lock
+#   pdm stores project-wide configurations in .pdm.toml, but it is recommended to not include it
+#   in version control.
+#   https://pdm.fming.dev/#use-with-ide
+.pdm.toml
+
+# PEP 582; used by e.g. github.com/David-OConnor/pyflow and github.com/pdm-project/pdm
+__pypackages__/
+
+# Celery stuff
+celerybeat-schedule
+celerybeat.pid
+
+# SageMath parsed files
+*.sage.py
+
+# Environments
+.env
+.venv
+env/
+venv/
+ENV/
+env.bak/
+venv.bak/
+
+# Spyder project settings
+.spyderproject
+.spyproject
+
+# Rope project settings
+.ropeproject
+
+# mkdocs documentation
+/site
+
+# mypy
+.mypy_cache/
+.dmypy.json
+dmypy.json
+
+# Pyre type checker
+.pyre/
+
+# pytype static type analyzer
+.pytype/
+
+# Cython debug symbols
+cython_debug/
+
+# PyCharm
+#  JetBrains specific template is maintained in a separate JetBrains.gitignore that can
+#  be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore
+#  and can be added to the global gitignore or merged into this file.  For a more nuclear
+#  option (not recommended) you can uncomment the following to ignore the entire idea folder.
+#.idea/
+
+# Ignore the folder with the vector database's data
+db/

README.md

@@ -1,13 +1,25 @@
----
-title: Langchain Test
-emoji: 🚀
-colorFrom: red
-colorTo: purple
-sdk: gradio
-sdk_version: 3.43.2
-app_file: app.py
-pinned: false
-license: mit
----
-
-Check out the configuration reference at https://huggingface.co/docs/hub/spaces-config-reference
+# Chat with a website
+
+Chat with a website using Apify and ChatGPT.
+
+## Setup
+
+Before getting started, be sure to sign up for an [Apify](https://console.apify.com/sign-up) and [OpenAI](https://openai.com/) account and create API keys.
+
+To set up and run this project, follow these steps:
+
+1. Install the required packages with `pip`:
+   ```
+   pip install -r requirements.txt
+   ```
+2. Rename the `.env.example` file to `.env` and replace the variables. Here's an explanation of the variables in the .env file:
+
+`OPENAI_API_KEY`: Your OpenAI API key. You can obtain it from your OpenAI account dashboard.  
+`APIFY_API_TOKEN`: Your Apify API token. You can obtain it from [Apify settings](https://console.apify.com/account/integrations).  
+`WEBSITE_URL`: The full URL of the website you'd like to chat with.  
+
+3. Run the `scrape.py` script to scrape the website's data using Apify's [Website content crawler](https://apify.com/apify/website-content-crawler).
+4. Run the Streamlit chat app, which should default to `http://localhost:8501` and allow you to chat with the website:
+   ```
+   streamlit run chat.py
+   ```

chat.py

@@ -0,0 +1,61 @@
+import os
+
+import streamlit as st
+from dotenv import load_dotenv
+from langchain.callbacks.base import BaseCallbackHandler
+from langchain.chains import ConversationalRetrievalChain
+from langchain.chat_models import ChatOpenAI
+from langchain.embeddings import OpenAIEmbeddings
+from langchain.memory import ConversationBufferMemory
+from langchain.memory.chat_message_histories import StreamlitChatMessageHistory
+from langchain.vectorstores import Chroma
+
+load_dotenv()
+
+website_url = os.environ.get('WEBSITE_URL', 'a website')
+
+st.set_page_config(page_title=f'Chat with {website_url}')
+st.title('Chat with a website')
+
+@st.cache_resource(ttl='1h')
+def get_retriever():
+    embeddings = OpenAIEmbeddings()
+    vectordb = Chroma(persist_directory='db', embedding_function=embeddings)
+
+    retriever = vectordb.as_retriever(search_type='mmr')
+
+    return retriever
+
+class StreamHandler(BaseCallbackHandler):
+    def __init__(self, container: st.delta_generator.DeltaGenerator, initial_text: str = ''):
+        self.container = container
+        self.text = initial_text
+
+    def on_llm_new_token(self, token: str, **kwargs) -> None:
+        self.text += token
+        self.container.markdown(self.text)
+
+retriever = get_retriever()
+
+msgs = StreamlitChatMessageHistory()
+memory = ConversationBufferMemory(memory_key='chat_history', chat_memory=msgs, return_messages=True)
+
+llm = ChatOpenAI(model_name='gpt-3.5-turbo', temperature=0, streaming=True)
+qa_chain = ConversationalRetrievalChain.from_llm(
+    llm, retriever=retriever, memory=memory, verbose=False
+)
+
+if st.sidebar.button('Clear message history') or len(msgs.messages) == 0:
+    msgs.clear()
+    msgs.add_ai_message(f'Ask me anything about {website_url}!')
+
+avatars = {'human': 'user', 'ai': 'assistant'}
+for msg in msgs.messages:
+    st.chat_message(avatars[msg.type]).write(msg.content)
+
+if user_query := st.chat_input(placeholder='Ask me anything!'):
+    st.chat_message('user').write(user_query)
+
+    with st.chat_message('assistant'):
+        stream_handler = StreamHandler(st.empty())
+        response = qa_chain.run(user_query, callbacks=[stream_handler])

requirements.txt

@@ -1,5 +1,7 @@
 apify-client
-langchain
 chromadb
-tiktoken
-openai
+langchain
+openai
+python-dotenv
+streamlit
+tiktoken

scrape.py

@@ -0,0 +1,40 @@
+import os
+
+from apify_client import ApifyClient
+from dotenv import load_dotenv
+from langchain.document_loaders import ApifyDatasetLoader
+from langchain.document_loaders.base import Document
+from langchain.embeddings.openai import OpenAIEmbeddings
+from langchain.text_splitter import RecursiveCharacterTextSplitter
+from langchain.vectorstores import Chroma
+
+# Load environment variables from a .env file
+load_dotenv()
+
+if __name__ == '__main__':
+    apify_client = ApifyClient(os.environ.get('APIFY_API_TOKEN'))
+    website_url = os.environ.get('WEBSITE_URL')
+    print(f'Extracting data from "{website_url}". Please wait...')
+    actor_run_info = apify_client.actor('apify/website-content-crawler').call(
+        run_input={'startUrls': [{'url': website_url}]}
+    )
+    print('Saving data into the vector database. Please wait...')
+    loader = ApifyDatasetLoader(
+        dataset_id=actor_run_info['defaultDatasetId'],
+        dataset_mapping_function=lambda item: Document(
+            page_content=item['text'] or '', metadata={'source': item['url']}
+        ),
+    )
+    documents = loader.load()
+    text_splitter = RecursiveCharacterTextSplitter(chunk_size=1500, chunk_overlap=100)
+    docs = text_splitter.split_documents(documents)
+
+    embedding = OpenAIEmbeddings()
+
+    vectordb = Chroma.from_documents(
+        documents=docs,
+        embedding=embedding,
+        persist_directory='db2',
+    )
+    vectordb.persist()
+    print('All done!')