Update app.py

app.py

@@ -31,6 +31,8 @@ INDEX_NAME_MULTILINGUAL = "budget-proposals-embeddinggemma"  # 768 dimensions fo
 # English: all-MiniLM-L6-v2 (better domain understanding)
 # Sinhala/Tamil: EmbeddingGemma-300m (better multilingual support)
 import os
+import re
+import google.generativeai as genai
 from huggingface_hub import login
 
 # Login to Hugging Face if token is available (for EmbeddingGemma)
@@ -38,6 +40,12 @@ hf_token = os.getenv('HF_TOKEN')
 if hf_token:
     login(token=hf_token)
 
+# Configure Gemini for transliteration
+gemini_api_key = os.getenv('GEMINI_API_KEY')
+if gemini_api_key:
+    genai.configure(api_key=gemini_api_key)
+    gemini_model = genai.GenerativeModel('gemini-2.5-flash')
+
 # Load both models
 embed_model_en = SentenceTransformer("all-MiniLM-L6-v2")
 embed_model_multilingual = SentenceTransformer("google/embeddinggemma-300m")
@@ -49,6 +57,53 @@ def get_embedding_model(language):
     else:  # si, ta, or any other language
         return embed_model_multilingual
 
+def contains_sinhala_roman(text):
+    """Check if text contains Roman Sinhala patterns"""
+    # Common Roman Sinhala patterns
+    sinhala_roman_patterns = [
+        r'\b[a-z]+[aeiou][a-z]*\b',  # Basic Sinhala roman patterns
+        r'\b(ma|ta|ka|ga|cha|ja|da|tha|pa|ba|ya|ra|la|wa|sa|ha|na|mata|kata|gata)\b',  # Common words
+    ]
+    
+    for pattern in sinhala_roman_patterns:
+        if re.search(pattern, text.lower()):
+            return True
+    return False
+
+def transliterate_sinhala_roman_to_sinhala(text):
+    """Use Gemini to convert Roman Sinhala to Sinhala script"""
+    if not gemini_api_key or not contains_sinhala_roman(text):
+        return text
+    
+    try:
+        prompt = f"""Convert this Roman Sinhala text to Sinhala script. Only convert if it's actually Sinhala words in Roman script. If it's English or other language, return as is.
+
+Text: "{text}"
+
+Sinhala script:"""
+        
+        response = gemini_model.generate_content(prompt)
+        result = response.text.strip()
+        
+        # Clean up the response
+        if result and len(result) > 0:
+            return result
+        else:
+            return text
+            
+    except Exception as e:
+        logger.warning(f"Transliteration failed: {e}")
+        return text
+
+def preprocess_query(query, language):
+    """Preprocess query with transliteration if needed"""
+    if language == 'si' and contains_sinhala_roman(query):
+        logger.info(f"Transliterating Roman Sinhala: {query}")
+        transliterated = transliterate_sinhala_roman_to_sinhala(query)
+        logger.info(f"Transliterated to: {transliterated}")
+        return transliterated
+    return query
+
 # Load dynamic metadata
 def load_dynamic_metadata():
     """Load metadata from dynamic_metadata.json"""
@@ -94,6 +149,10 @@ def semantic_search(query: str, top_k=1, category_filter=None, language='en'):
         global DYNAMIC_METADATA
         DYNAMIC_METADATA = load_dynamic_metadata()
         
+        # Preprocess query with transliteration if needed
+        original_query = query
+        query = preprocess_query(query, language)
+        
         pc_index = get_pinecone_index(language)
         if not pc_index:
             return []