First commit

.gitignore

@@ -0,0 +1,4 @@
+*/__pycache__/*
+__pycache__
+
+.env

app.py

@@ -0,0 +1,45 @@
+import streamlit as st
+import dotenv
+import os
+
+from rag import Rag
+from vectore_store.PineconeConnector import PineconeConnector
+from vectore_store.VectoreStoreManager import VectoreStoreManager
+
+GROUP_NAME = "Groupe 1"
+
+def main():
+        
+    if len(st.session_state) == 0:
+        # Define Vectore store strategy
+        pinecone_connector = PineconeConnector()
+        vs_manager = VectoreStoreManager(pinecone_connector)
+
+        st.session_state["messages"] = []
+        st.session_state["assistant"] = Rag(vectore_store=vs_manager)
+
+    st.set_page_config(page_title=GROUP_NAME)
+
+    st.title(GROUP_NAME)
+
+    prompt_system = st.Page("pages/prompt_system.py", title="Prompt système", icon="📋", default=True)
+    saved_documents = st.Page("pages/persistent_documents.py", title="Documents Communs", icon="📋")
+    documents = st.Page("pages/documents.py", title="Documents", icon="📋")
+    form = st.Page("pages/form.py", title="Formulaire", icon="📋")
+    chatbot = st.Page("pages/chatbot.py", title="Chatbot", icon="📋")
+
+    pg = st.navigation(
+        [
+            saved_documents,
+            prompt_system,
+            documents,
+            form,
+            chatbot
+        ]
+    )
+
+    pg.run()
+
+
+if __name__ == "__main__":
+    main()

pages/chatbot.py

@@ -0,0 +1,36 @@
+import streamlit as st
+from streamlit_chat import message
+
+def display_messages():
+    for i, (msg, is_user) in enumerate(st.session_state["messages"]):
+        message(msg, is_user=is_user, key=str(i))
+    st.session_state["thinking_spinner"] = st.empty()
+
+
+def process_input():
+    if st.session_state["user_input"] and len(st.session_state["user_input"].strip()) > 0:
+        user_text = st.session_state["user_input"].strip()
+
+
+        with st.session_state["thinking_spinner"], st.spinner(f"Je réfléchis"):
+            agent_text = st.session_state["assistant"].ask(user_text, st.session_state["messages"] if "messages" in st.session_state else [])
+
+        st.session_state["messages"].append((user_text, True))
+        st.session_state["messages"].append((agent_text, False))
+
+
+def page():
+    st.subheader("Posez vos questions")
+
+    if "assistant" not in st.session_state:
+        st.text("Assistant non initialisé")
+
+
+    prompt_sys = st.session_state.prompt_system if 'prompt_system' in st.session_state and st.session_state.prompt_system != '' else "Renseignez votre prompt system"
+
+    st.text("Prompt system : " + prompt_sys)
+
+    display_messages()
+    st.text_input("Message", key="user_input", on_change=process_input)
+
+page()

pages/documents.py

@@ -0,0 +1,35 @@
+import os
+import tempfile
+import streamlit as st
+
+def read_and_save_file():
+    st.session_state["messages"] = []
+    st.session_state["user_input"] = ""
+
+    for file in st.session_state["file_uploader"]:
+        with tempfile.NamedTemporaryFile(delete=False) as tf:
+            tf.write(file.getbuffer())
+            file_path = tf.name
+
+        with st.session_state["ingestion_spinner"], st.spinner(f"Chargement {file.name}"):
+            st.session_state["assistant"].ingest(file_path)
+        os.remove(file_path)
+
+
+
+def page():
+    st.subheader("Charger vos documents")
+
+    # File uploader
+    uploaded_file = st.file_uploader(
+        "Télécharger un ou plusieurs documents",
+        type=["pdf"],
+        key="file_uploader",
+        accept_multiple_files=True,
+        on_change=read_and_save_file,
+    )
+
+
+    st.session_state["ingestion_spinner"] = st.empty()
+
+page()

pages/form.py

@@ -0,0 +1,6 @@
+import streamlit as st
+
+def page():
+    st.subheader("Définissez vos paramètres")
+
+page()

pages/persistent_documents.py

@@ -0,0 +1,35 @@
+import os
+import tempfile
+import streamlit as st
+
+def uploadToDb():
+
+    for file in st.session_state["file_uploader_commun"]:
+        with tempfile.NamedTemporaryFile(delete=False) as tf:
+            tf.write(file.getbuffer())
+            file_path = tf.name
+
+        with st.session_state["ingestion_spinner"], st.spinner(f"Chargement {file.name}"):
+            st.session_state["assistant"].ingestToDb(file_path, filename=file.name)
+        os.remove(file_path)
+
+def page():
+    st.subheader("Montez des documents communs")
+    
+    st.file_uploader(
+        "Télécharger un documents",
+        type=["pdf"],
+        key="file_uploader_commun",
+        accept_multiple_files=True,
+        on_change=uploadToDb,
+    )
+
+    st.session_state["ingestion_spinner"] = st.empty()
+
+    st.divider()
+    st.write("Documents dans la base de données", bold=True)
+    
+    for doc in st.session_state["assistant"].vector_store.getDocs():
+        st.write(" - "+doc)
+    
+page()

pages/prompt_system.py

@@ -0,0 +1,12 @@
+import streamlit as st
+
+def page():
+
+    st.subheader("Renseignez votre prompt system")
+
+    prompt = st.text_area("Prompt system", st.session_state.prompt_system if 'prompt_system' in st.session_state else "")
+
+    # Session State also supports attribute based syntax
+    st.session_state['prompt_system'] = prompt
+
+page()

prompt_template.py

@@ -0,0 +1,8 @@
+base_template = '''
+
+Documents partagées : {commonContext}
+Document de référence : {documentContext}
+
+Voici l'historique des messages : {messages}
+Les attentes de l'utilisateur sont : {query}
+'''

rag.py

@@ -0,0 +1,100 @@
+import os
+
+from dotenv import load_dotenv
+from langchain_community.vectorstores import FAISS
+from langchain_mistralai.chat_models import ChatMistralAI
+from langchain_mistralai.embeddings import MistralAIEmbeddings
+from langchain.schema.output_parser import StrOutputParser
+from langchain_community.document_loaders import PyPDFLoader
+from langchain.text_splitter import RecursiveCharacterTextSplitter
+from langchain.schema.runnable import RunnablePassthrough
+from langchain.prompts import PromptTemplate
+from langchain_community.vectorstores.utils import filter_complex_metadata
+#add new import
+from langchain_community.document_loaders.csv_loader import CSVLoader
+
+from prompt_template import base_template
+
+
+# load .env in local dev
+load_dotenv()
+env_api_key = os.environ.get("MISTRAL_API_KEY")
+llm_model = "open-mixtral-8x7b"
+
+class Rag:
+    document_vector_store = None
+    retriever = None
+    chain = None
+
+    def __init__(self, vectore_store=None):
+        
+        self.model = ChatMistralAI(model=llm_model)
+        self.embedding = MistralAIEmbeddings(model="mistral-embed", mistral_api_key=env_api_key)
+
+        self.text_splitter = RecursiveCharacterTextSplitter(chunk_size=500, chunk_overlap=100, length_function=len)
+        self.prompt = PromptTemplate.from_template(base_template)
+
+        self.vector_store = vectore_store
+
+    def setModel(self, model):
+        self.model = model
+
+    def ingestToDb(self, file_path: str, filename: str):
+
+        docs = PyPDFLoader(file_path=file_path).load()
+
+        # Extract all text from the document
+        text = ""
+        for page in docs:
+            text += page.page_content
+
+        # Split the text into chunks
+        chunks = self.text_splitter.split_text(text)
+        
+        return self.vector_store.addDoc(filename=filename, text_chunks=chunks, embedding=self.embedding)
+
+    def getDbFiles(self):
+        return self.vector_store.getDocs()
+
+    def ingest(self, pdf_file_path: str):
+        docs = PyPDFLoader(file_path=pdf_file_path).load()
+       
+        chunks = self.text_splitter.split_documents(docs)
+        chunks = filter_complex_metadata(chunks)
+
+        document_vector_store = FAISS.from_documents(chunks, self.embedding)
+        
+        self.retriever = document_vector_store.as_retriever(
+            search_type="similarity_score_threshold",
+            search_kwargs={
+                "k": 3,
+                "score_threshold": 0.5,
+            },
+        )
+
+        self.chain = self.prompt | self.model | StrOutputParser()
+
+    def ask(self, query: str, messages: list):
+        if not self.chain:
+            return "Ajouter un document PDF d'abord."
+        
+        print("messages ", messages)
+
+        # Retrieve the context document
+        documentContext = self.retriever.invoke(query)
+
+        # Retrieve the VectoreStore
+        contextCommon = None
+
+        return self.chain.invoke({
+            "query": query,
+            "documentContext": documentContext,
+            "commonContext": contextCommon,
+            "messages": messages
+        })
+
+    def clear(self):
+        self.document_vector_store = None
+        self.vector_store = None
+        self.retriever = None
+        self.chain = None

requirements.txt

@@ -0,0 +1,19 @@
+streamlit==1.37.0
+streamlit_chat
+# abc
+python-dotenv
+pymupdf
+python-multipart
+pydantic
+langchain-pinecone 
+pinecone-notebooks
+pinecone-client[grpc]
+async-timeout
+pymupdf
+python-dotenv
+typing-extensions
+langchain
+langchain-openai
+langchain-community
+langchain-pinecone
+langchain_mistralai

vectore_store/ConnectorStrategy.py

@@ -0,0 +1,14 @@
+from abc import ABC, abstractmethod
+
+class ConnectorStrategy(ABC):
+    @abstractmethod
+    def getDocs(self):
+        pass
+    
+    @abstractmethod
+    def addDoc(self, filename, text_chunks, embedding):
+        pass
+    
+    @abstractmethod
+    def retriever(self, query, embedding):
+        pass

vectore_store/PineconeConnector.py

@@ -0,0 +1,110 @@
+import os 
+from dotenv import load_dotenv
+
+from .ConnectorStrategy import ConnectorStrategy
+
+from pinecone import Pinecone, ServerlessSpec
+from langchain_openai import OpenAIEmbeddings
+from langchain_pinecone import PineconeVectorStore
+from langchain_core.documents import Document
+
+import unicodedata
+import time
+
+class PineconeConnector(ConnectorStrategy):
+    def __init__(self):
+
+        load_dotenv()
+
+        pinecone_api_key = os.environ.get("PINECONE_API_KEY")
+
+        self.index_name = os.environ.get("PINECONE_INDEX_NAME")
+        self.namespace = os.environ.get("PINECONE_NAMESPACE")
+
+        print(f"Index name: {self.index_name}")
+        print(f"Namespace: {self.namespace}")
+        print(f"Pinecone API Key: {pinecone_api_key}")
+
+        pc = Pinecone(api_key=pinecone_api_key)
+
+        existing_indexes = [index_info["name"] for index_info in pc.list_indexes()]
+
+        if self.index_name not in existing_indexes:
+            pc.create_index(
+                name=self.index_name,
+                dimension=3072,
+                metric="cosine",
+                spec=ServerlessSpec(cloud="aws", region="us-east-1"),
+            )
+            while not pc.describe_index(self.index_name).status["ready"]:
+                time.sleep(1)
+
+        self.index = pc.Index(self.index_name)
+
+    
+    def getDocs(self):
+        # Simulate getting docs from Pinecone
+        print("Fetching documents from Pinecone")
+        
+        docs_names = []
+        for ids in self.index.list(namespace=self.namespace):
+            for id in ids:
+                name_doc = "_".join(id.split("_")[:-1])
+                if name_doc not in docs_names:
+                    docs_names.append(name_doc)
+
+        return docs_names
+    
+    
+    def addDoc(self, filename, text_chunks, embedding):
+        try:
+            vector_store = PineconeVectorStore(index=self.index, embedding=embedding,namespace=self.namespace)
+
+            file_name = filename.split(".")[0].replace(" ","_").replace("-","_").replace(".","_").replace("/","_").replace("\\","_").strip()
+
+            documents = []
+            uuids = []
+
+            print(file_name)
+
+            for i, chunk in enumerate(text_chunks):
+                clean_filename = remove_non_standard_ascii(file_name)
+                uuid = f"{clean_filename}_{i}"
+                
+                print(f"Adding document with ID {uuid}")
+
+                document = Document(
+                    page_content=chunk,
+                    metadata={ "filename":filename, "chunk_id":uuid },
+                )
+
+                uuids.append(uuid)
+                documents.append(document)
+            
+
+            vector_store.add_documents(documents=documents, ids=uuids)
+
+            return {"filename_id":clean_filename}
+        
+        except Exception as e:
+            print(e)
+            return False
+    
+    def retriever(self, query, embedding):
+
+        print(f"Retrieving documents from Pinecone for query '{query}'")
+
+        vector_store = PineconeVectorStore(index=self.index, embedding=embedding,namespace=self.namespace)
+
+        retriever = vector_store.as_retriever(
+            search_type="similarity_score_threshold",
+            search_kwargs={"k": 3, "score_threshold": 0.6},
+        )
+
+        return retriever.invoke(query)
+    
+
+def remove_non_standard_ascii(input_string: str) -> str:
+    normalized_string = unicodedata.normalize('NFKD', input_string)
+    return ''.join(char for char in normalized_string if 'a' <= char <= 'z' or 'A' <= char <= 'Z' or char.isdigit() or char in ' .,!?')
+

vectore_store/VectoreStoreManager.py

@@ -0,0 +1,15 @@
+from vectore_store import ConnectorStrategy
+
+
+class VectoreStoreManager:
+    def __init__(self, strategy: ConnectorStrategy):
+        self.strategy = strategy
+    
+    def getDocs(self):
+        return self.strategy.getDocs()
+    
+    def addDoc(self, filename, text_chunks, embedding):
+        self.strategy.addDoc(filename, text_chunks, embedding)
+    
+    def retriever(self, query, embedding):
+        return self.strategy.retriever(query, embedding)