added bm25 +history

app.py

@@ -68,7 +68,7 @@ for idx, row in df.iterrows():
         )
 
 # ---------------------- Config ----------------------
-SIMILARITY_THRESHOLD = 0.80
+SIMILARITY_THRESHOLD = 0.75
 client1 = OpenAI(base_url="https://openrouter.ai/api/v1", api_key=API_KEY)  # Replace with your OpenRouter API key
 
 # ---------------------- Models ----------------------
@@ -81,6 +81,19 @@ with open("qa.json", "r", encoding="utf-8") as f:
 qa_questions = list(qa_data.keys())
 qa_answers = list(qa_data.values())
 qa_embeddings = semantic_model.encode(qa_questions, convert_to_tensor=True)
+#-------------------------bm25---------------------------------
+from rank_bm25 import BM25Okapi
+from nltk.tokenize import word_tokenize
+
+def rerank_with_bm25(docs, query):
+    tokenized_docs = [word_tokenize(doc['content'].lower()) for doc in docs]
+    bm25 = BM25Okapi(tokenized_docs)
+    tokenized_query = word_tokenize(query.lower())
+
+    scores = bm25.get_scores(tokenized_query)
+    top_indices = sorted(range(len(scores)), key=lambda i: scores[i], reverse=True)[:3]
+    return [docs[i] for i in top_indices]
+
 
 # ---------------------- History-Aware CAG ----------------------
 def retrieve_from_cag(user_query, chat_history):
@@ -97,18 +110,19 @@ def retrieve_from_cag(user_query, chat_history):
 
 # ---------------------- History-Aware RAG ----------------------
 def retrieve_from_rag(user_query, chat_history):
-    # Combine the previous chat history with the current query for context
+    # Combine history with current query
     history_context = " ".join([f"User: {msg[0]} Bot: {msg[1]}" for msg in chat_history]) + " "
     full_query = history_context + user_query
 
     print("Searching in RAG with history context...")
 
     query_embedding = embedding_model.encode(full_query)
-    results = collection.query(query_embeddings=[query_embedding], n_results=3)
+    results = collection.query(query_embeddings=[query_embedding], n_results=5)  # Get top 5 first
 
     if not results or not results.get('documents'):
         return None
 
+    # Build docs list
     documents = []
     for i, content in enumerate(results['documents'][0]):
         metadata = results['metadatas'][0][i]
@@ -116,8 +130,12 @@ def retrieve_from_rag(user_query, chat_history):
             "content": content.strip(),
             "metadata": metadata
         })
-    print("Documents retrieved:", documents)
-    return documents
+
+    #  Rerank with BM25
+    top_docs = rerank_with_bm25(documents, user_query)
+
+    print("BM25-selected top 3 documents:", top_docs)
+    return top_docs
 
 # ---------------------- Generation function (OpenRouter) ----------------------
 def generate_via_openrouter(context, query, chat_history=None):