commat cammit 1

app.py

@@ -1,4 +1,58 @@
+import os
 import streamlit as st
+from src.indexing.document_processor import DocumentProcessor
+from src.indexing.vectore_store import VectorStoreManager
+from src.retriever.retriever import RetrieverManager
 
-x = st.slider('Select a value')
-st.write(x, 'squared is', x * x)
+UPLOAD_FOLDER = "uploads/"
+PERSIST_DIRECTORY = "chroma_db/"
+os.makedirs(UPLOAD_FOLDER, exist_ok=True)
+os.makedirs(PERSIST_DIRECTORY, exist_ok=True)
+
+if "messages" not in st.session_state:
+    st.session_state.messages = []
+if "retriever" not in st.session_state:
+    st.session_state.retriever = None
+if "vector_store" not in st.session_state:
+    st.session_state.vector_store = None
+
+st.set_page_config(
+    page_title="RAG Chatbot",
+    layout="wide",
+    page_icon="📘",
+)
+st.title("Agentic RAG Chatbot")
+
+
+with st.sidebar:
+    st.header("PDF Upload")
+    uploaded_file = st.file_uploader("Upload your PDF", type=["pdf"])
+    st.info("Supported file type: PDF")
+
+    if uploaded_file:
+        with st.spinner("Processing PDF..."):
+            
+            file_path = os.path.join(UPLOAD_FOLDER, uploaded_file.name)
+            with open(file_path, "wb") as f:
+                f.write(uploaded_file.getbuffer())
+
+            
+            doc_processor = DocumentProcessor()
+            chunks = doc_processor.load_and_split_pdf(file_path)
+
+            # Buat vector store
+            vector_store_manager = VectorStoreManager()
+            vector_store = vector_store_manager.index_documents(
+                documents=chunks,
+                collection_name=uploaded_file.name,
+                persist_directory=PERSIST_DIRECTORY
+            )
+            st.session_state.vector_store = vector_store
+
+            # Setup retriever
+            retriever_manager = RetrieverManager(vector_store)
+            base_retriever = retriever_manager.create_base_retriever()
+            compression_retriever = retriever_manager.create_compression_retriever(base_retriever)
+            st.session_state.retriever = compression_retriever
+
+            st.success("File processed successfully!")

requirements.txt

@@ -0,0 +1,11 @@
+langchain 
+langgraph
+langchain-huggingface 
+langchain-google-genai 
+google-ai-generativelanguage==0.6.15
+langchain-community 
+langchain-chroma
+pypdf 
+tiktoken 
+rank_bm25 
+flashrank

src/indexing/document_processor.py

@@ -0,0 +1,16 @@
+from langchain.text_splitter import RecursiveCharacterTextSplitter
+from langchain.document_loaders import PyPDFLoader
+
+class DocumentProcessor:
+    def __init__(self, chunk_size=500, chunk_overlap=100):
+        self.text_splitter = RecursiveCharacterTextSplitter(
+            chunk_size=chunk_size, 
+            chunk_overlap=chunk_overlap
+        )
+
+    def load_and_split_pdf(self, file_path: str):
+        """Load PDF and split into chunks"""
+        loader = PyPDFLoader(file_path)
+        docs = loader.load()
+        chunks = self.text_splitter.split_documents(docs)
+        return chunks

src/indexing/vectore_store.py

@@ -0,0 +1,23 @@
+from langchain_huggingface import HuggingFaceEmbeddings
+from langchain_chroma import Chroma
+
+class VectorStoreManager:
+    def __init__(self, embedding_model="intfloat/multilingual-e5-small"):
+        self.embeddings = HuggingFaceEmbeddings(model_name=embedding_model)
+        
+    def create_vector_store(self, collection_name="my_collection", persist_directory=None):
+        """Create a new vector store"""
+        store_params = {
+            "collection_name": collection_name,
+            "embedding_function": self.embeddings,
+        }
+        if persist_directory:
+            store_params["persist_directory"] = persist_directory
+            
+        return Chroma(**store_params)
+    
+    def index_documents(self, documents, collection_name="my_collection", persist_directory=None):
+        """Index documents into vector store"""
+        vector_store = self.create_vector_store(collection_name, persist_directory)
+        vector_store.add_documents(documents=documents)
+        return vector_store

src/retriever/retriever.py

@@ -0,0 +1,34 @@
+from langchain.retrievers import BM25Retriever, EnsembleRetriever
+from langchain.retrievers import ContextualCompressionRetriever
+from langchain.retrievers.document_compressors import FlashrankRerank
+
+
+class RetrieverManager:
+    def __init__(self, vector_store):
+        self.vector_store = vector_store
+        
+    def create_base_retriever(self, search_type="similarity", k=3):
+        """Create basic vector store retriever"""
+        return self.vector_store.as_retriever(
+            search_type=search_type, 
+            search_kwargs={"k": k}
+        )
+    
+    def create_ensemble_retriever(self, texts, vector_weight=0.5, keyword_weight=0.5):
+        """Create ensemble retriever combining vector and keyword search"""
+        vector_retriever = self.create_base_retriever()
+        keyword_retriever = BM25Retriever.from_documents(texts)
+        keyword_retriever.k = 3
+        
+        return EnsembleRetriever(
+            retrievers=[vector_retriever, keyword_retriever],
+            weights=[vector_weight, keyword_weight]
+        )
+    
+    def create_compression_retriever(self, base_retriever, top_n=5):
+        """Create compression retriever with reranking"""
+        compressor = FlashrankRerank(top_n=top_n)
+        return ContextualCompressionRetriever(
+            base_compressor=compressor,
+            base_retriever=base_retriever
+        )