Update app.py

app.py

@@ -5,6 +5,7 @@ from pinecone import Pinecone
 import os
 import logging
 import json
+from env_config import PINECONE_API_KEY
 
 app = Flask(__name__)
 CORS(app)  # Enable CORS for all routes
@@ -14,13 +15,9 @@ logging.basicConfig(level=logging.INFO)
 logger = logging.getLogger(__name__)
 
 # Initialize Pinecone
-PINECONE_API_KEY = os.getenv('PINECONE_API_KEY')
-if not PINECONE_API_KEY:
-    raise ValueError("PINECONE_API_KEY environment variable is required")
-
 pc = Pinecone(api_key=PINECONE_API_KEY)
 # Configuration
-INDEX_NAME = "budget-proposals"  # Use the new optimized index
+INDEX_NAME = "budget-proposals-optimized"  # Use the new optimized index
 
 # Load embedding model
 embed_model = SentenceTransformer("sentence-transformers/all-MiniLM-L6-v2")
@@ -36,9 +33,22 @@ def load_dynamic_metadata():
         logger.error(f"Error loading dynamic metadata: {e}")
     return {}
 
-# Load dynamic metadata
+# Load dynamic metadata (will be reloaded on each request)
 DYNAMIC_METADATA = load_dynamic_metadata()
 
+def get_language_specific_data(proposal_data, field, language='en'):
+    """Get language-specific data from proposal metadata"""
+    # If it's the old format (single language), return as-is
+    if isinstance(proposal_data.get(field), str):
+        return proposal_data.get(field, '')
+    
+    # If it's the new multi-language format, return language-specific data
+    if isinstance(proposal_data.get(field), dict):
+        return proposal_data.get(field, {}).get(language, 
+                                               proposal_data.get(field, {}).get('en', ''))
+    
+    return ''
+
 def get_pinecone_index():
     """Get the budget proposals Pinecone index"""
     try:
@@ -47,9 +57,13 @@ def get_pinecone_index():
         logger.error(f"Error accessing Pinecone index: {e}")
         return None
 
-def semantic_search(query: str, top_k=1, category_filter=None):
-    """Perform semantic search on budget proposals - return relevant documents based on query specificity"""
+def semantic_search(query: str, top_k=1, category_filter=None, language='en'):
+    """Perform semantic search on budget proposals with multi-language support"""
     try:
+        # Reload metadata to get latest updates
+        global DYNAMIC_METADATA
+        DYNAMIC_METADATA = load_dynamic_metadata()
+        
         pc_index = get_pinecone_index()
         if not pc_index:
             return []
@@ -87,28 +101,24 @@ def semantic_search(query: str, top_k=1, category_filter=None):
         # Sort documents by their best scores
         sorted_docs = sorted(best_scores.items(), key=lambda x: x[1], reverse=True)
         
-       
         # Determine how many documents to return based on query specificity
         max_score = sorted_docs[0][1]  # Best score
         
-        # Minimum threshold - if best score is too low, return no results
-        MIN_SCORE_THRESHOLD = 0.05  # Adjust this value as needed
-        
+        # If the best score is very high (>0.6), it's a specific query - show fewer results
+        # If the best score is moderate (0.3-0.6), it's a medium query - show some results
+        # If the best score is low (<0.3), it's a broad query - show more results
         if max_score > 0.6:
-         # Specific query - show 1-2 documents
-            threshold = max_score * 0.8
+            # Specific query - show 1-2 documents
+            threshold = max_score * 0.8  # Show documents within 80% of best score
             max_docs = 2
         elif max_score > 0.3:
-        # Medium query - show 2-3 documents  
-            threshold = max_score * 0.7
+            # Medium query - show 2-3 documents
+            threshold = max_score * 0.7  # Show documents within 70% of best score
             max_docs = 3
-        elif max_score >= MIN_SCORE_THRESHOLD:  # 0.3+ range
+        else:
             # Broad query - show 3-5 documents
-            threshold = max_score * 0.5
+            threshold = max_score * 0.5  # Show documents within 50% of best score
             max_docs = 5
-        else:
-            # Score too low - return no results
-            return []
         
         results = []
         doc_count = 0
@@ -129,10 +139,11 @@ def semantic_search(query: str, top_k=1, category_filter=None):
                         "costLKR": metadata.get("costLKR", "No Costing Available")
                     })
                     
-                    title = proposal_data["title"]
-                    summary = proposal_data["summary"]
-                    costLKR = proposal_data["costLKR"]
-                    category = proposal_data["category"]
+                    # Get language-specific data
+                    title = get_language_specific_data(proposal_data, "title", language)
+                    summary = get_language_specific_data(proposal_data, "summary", language)
+                    costLKR = get_language_specific_data(proposal_data, "costLKR", language)
+                    category = get_language_specific_data(proposal_data, "category", language)
                     thumb_url = metadata.get("thumbUrl", "")
                     
                     result = {
@@ -158,9 +169,13 @@ def semantic_search(query: str, top_k=1, category_filter=None):
         logger.error(f"Search error: {e}")
         return []
 
-def get_all_proposals(category_filter=None):
-    """Get all budget proposals (for initial load or when no search query)"""
+def get_all_proposals(category_filter=None, language='en'):
+    """Get all budget proposals with multi-language support"""
     try:
+        # Reload metadata to get latest updates
+        global DYNAMIC_METADATA
+        DYNAMIC_METADATA = load_dynamic_metadata()
+        
         pc_index = get_pinecone_index()
         if not pc_index:
             logger.warning("Pinecone index not available, returning empty list")
@@ -204,26 +219,32 @@ def get_all_proposals(category_filter=None):
                 "costLKR": metadata.get("costLKR", "No Costing Available")
             })
 
-            title = proposal_data["title"]
-            summary = proposal_data["summary"]
-            costLKR = proposal_data["costLKR"]
-            category = proposal_data["category"]
+            # Get language-specific data
+            title = get_language_specific_data(proposal_data, "title", language)
+            summary = get_language_specific_data(proposal_data, "summary", language)
+            costLKR = get_language_specific_data(proposal_data, "costLKR", language)
+            category = get_language_specific_data(proposal_data, "category", language)
             thumb_url = metadata.get("thumbUrl", "")
             
-            result = {
-                "title": title,
-                "summary": summary,
-                "costLKR": costLKR,
-                "category": category,
-                "pdfUrl": f"assets/pdfs/{file_path}" if file_path else "",
-                "thumbUrl": f"assets/thumbs/{thumb_url}" if thumb_url else "",
-                "score": 1.0,  # Default score for all proposals
-                "relevance_percentage": 100,
-                "file_path": file_path,
-                "id": match["id"]
-            }
-            
-            results.append(result)
+            # Only include documents that have meaningful content in the requested language
+            # Skip documents where title and summary are empty or "Unknown"/"No summary available"
+            if (title and title.strip() and title not in ["Unknown", "Unknown Title"] and
+                summary and summary.strip() and summary not in ["No summary available", ""]):
+                
+                result = {
+                    "title": title,
+                    "summary": summary,
+                    "costLKR": costLKR,
+                    "category": category,
+                    "pdfUrl": f"assets/pdfs/{file_path}" if file_path else "",
+                    "thumbUrl": f"assets/thumbs/{thumb_url}" if thumb_url else "",
+                    "score": 1.0,  # Default score for all proposals
+                    "relevance_percentage": 100,
+                    "file_path": file_path,
+                    "id": match["id"]
+                }
+                
+                results.append(result)
         
         return results
         
@@ -233,24 +254,26 @@ def get_all_proposals(category_filter=None):
 
 @app.route('/api/search', methods=['POST'])
 def search_proposals():
-    """API endpoint for searching budget proposals"""
+    """API endpoint for searching budget proposals with multi-language support"""
     try:
         data = request.get_json()
         query = data.get('query', '').strip()
         top_k = data.get('top_k', 10)
         category_filter = data.get('category_filter')
+        language = data.get('language', 'en')  # Default to English
         
         if not query:
             # If no query, return all proposals
-            results = get_all_proposals(category_filter)
+            results = get_all_proposals(category_filter, language)
         else:
-            results = semantic_search(query, top_k, category_filter)
+            results = semantic_search(query, top_k, category_filter, language)
         
         return jsonify({
             "query": query,
             "results": results,
             "total_results": len(results),
-            "category_filter": category_filter
+            "category_filter": category_filter,
+            "language": language
         })
     
     except Exception as e:
@@ -259,23 +282,25 @@ def search_proposals():
 
 @app.route('/api/search', methods=['GET'])
 def search_proposals_get():
-    """API endpoint for searching proposals (GET method)"""
+    """API endpoint for searching proposals (GET method) with multi-language support"""
     try:
         query = request.args.get('query', '').strip()
         top_k = int(request.args.get('top_k', 10))
         category_filter = request.args.get('category_filter')
+        language = request.args.get('language', 'en')  # Default to English
         
         if not query:
             # If no query, return all proposals
-            results = get_all_proposals(category_filter)
+            results = get_all_proposals(category_filter, language)
         else:
-            results = semantic_search(query, top_k, category_filter)
+            results = semantic_search(query, top_k, category_filter, language)
         
         return jsonify({
             "query": query,
             "results": results,
             "total_results": len(results),
-            "category_filter": category_filter
+            "category_filter": category_filter,
+            "language": language
         })
     
     except Exception as e:
@@ -284,15 +309,17 @@ def search_proposals_get():
 
 @app.route('/api/proposals', methods=['GET'])
 def get_proposals():
-    """Get all budget proposals"""
+    """Get all budget proposals with multi-language support"""
     try:
         category_filter = request.args.get('category_filter')
-        results = get_all_proposals(category_filter)
+        language = request.args.get('language', 'en')  # Default to English
+        results = get_all_proposals(category_filter, language)
         
         return jsonify({
             "results": results,
             "total_results": len(results),
-            "category_filter": category_filter
+            "category_filter": category_filter,
+            "language": language
         })
     
     except Exception as e: