Update app.py

app.py

@@ -157,59 +157,19 @@ Converted Tamil script:"""
         logger.warning(f"Tamil transliteration failed: {e}")
         return text
 
-def detect_language_from_query(query):
-    """Detect language from query text"""
-    # Check for Sinhala Unicode
-    if re.search(r'[\u0D80-\u0DFF]', query):
-        return 'si'
-    
-    # Check for Tamil Unicode
-    if re.search(r'[\u0B80-\u0BFF]', query):
-        return 'ta'
-    
-    # Check for Singlish (Romanized Sinhala)
-    if contains_sinhala_roman(query):
-        return 'singlish'
-    
-    # Check for Romanized Tamil
-    if contains_tamil_roman(query):
-        return 'romanized_tamil'
-    
-    # Default to English
-    return 'en'
-
 def preprocess_query(query, language):
     """Preprocess query with transliteration if needed"""
-    # Auto-detect language if not explicitly provided or if it's 'en' (default)
-    if language == 'en' or not language:
-        detected_language = detect_language_from_query(query)
-        logger.info(f"Auto-detected language: {detected_language} for query: {query}")
-    else:
-        detected_language = language
-    
-    # Handle Singlish (Romanized Sinhala) - always transliterate to Sinhala script
-    if detected_language == 'singlish' or (language == 'si' and contains_sinhala_roman(query)):
-        logger.info(f"Transliterating Singlish to Sinhala: {query}")
+    if language == 'si' and contains_sinhala_roman(query):
+        logger.info(f"Transliterating Roman Sinhala: {query}")
         transliterated = transliterate_sinhala_roman_to_sinhala(query)
         logger.info(f"Transliterated to: {transliterated}")
-        return transliterated, 'si'  # Return both transliterated text and target language
-    
-    # Handle Romanized Tamil - always transliterate to Tamil script
-    elif detected_language == 'romanized_tamil' or (language == 'ta' and contains_tamil_roman(query)):
-        logger.info(f"Transliterating Romanized Tamil to Tamil: {query}")
+        return transliterated
+    elif language == 'ta' and contains_tamil_roman(query):
+        logger.info(f"Transliterating Roman Tamil: {query}")
         transliterated = transliterate_tamil_roman_to_tamil(query)
         logger.info(f"Transliterated to: {transliterated}")
-        return transliterated, 'ta'  # Return both transliterated text and target language
-    
-    # For proper Sinhala/Tamil Unicode, use as-is
-    elif detected_language in ['si', 'ta']:
-        logger.info(f"Using original {detected_language} text: {query}")
-        return query, detected_language
-    
-    # For English, use as-is
-    else:
-        logger.info(f"Using original English text: {query}")
-        return query, 'en'
+        return transliterated
+    return query
 
 # Load dynamic metadata
 def load_dynamic_metadata():
@@ -258,17 +218,14 @@ def semantic_search(query: str, top_k=1, category_filter=None, language='en'):
         
         # Preprocess query with transliteration if needed
         original_query = query
-        query, detected_language = preprocess_query(query, language)
+        query = preprocess_query(query, language)
         
-        # Use the detected language for embedding model and index selection
-        search_language = detected_language
-        
-        pc_index = get_pinecone_index(search_language)
+        pc_index = get_pinecone_index(language)
         if not pc_index:
             return []
         
         # Use language-specific embedding model
-        model = get_embedding_model(search_language)
+        model = get_embedding_model(language)
         query_emb = model.encode(query).tolist()
         
         # Build filter if category is specified
@@ -340,11 +297,11 @@ def semantic_search(query: str, top_k=1, category_filter=None, language='en'):
                         "costLKR": metadata.get("costLKR", "No Costing Available")
                     })
                     
-                    # Get language-specific data using the detected language
-                    title = get_language_specific_data(proposal_data, "title", search_language)
-                    summary = get_language_specific_data(proposal_data, "summary", search_language)
-                    costLKR = get_language_specific_data(proposal_data, "costLKR", search_language)
-                    category = get_language_specific_data(proposal_data, "category", search_language)
+                    # Get language-specific data
+                    title = get_language_specific_data(proposal_data, "title", language)
+                    summary = get_language_specific_data(proposal_data, "summary", language)
+                    costLKR = get_language_specific_data(proposal_data, "costLKR", language)
+                    category = get_language_specific_data(proposal_data, "category", language)
                     thumb_url = metadata.get("thumbUrl", "")
                     
                     # Only include documents that have meaningful content in the requested language
@@ -370,22 +327,10 @@ def semantic_search(query: str, top_k=1, category_filter=None, language='en'):
                         doc_count += 1
                     break
         
-        return {
-            "results": results,
-            "original_query": original_query,
-            "processed_query": query,
-            "detected_language": search_language,
-            "transliterated": original_query != query
-        }
+        return results
     except Exception as e:
         logger.error(f"Search error: {e}")
-        return {
-            "results": [],
-            "original_query": query,
-            "processed_query": query,
-            "detected_language": language,
-            "transliterated": False
-        }
+        return []
 
 def get_all_proposals(category_filter=None, language='en'):
     """Get all budget proposals with multi-language support"""
@@ -486,28 +431,16 @@ def search_proposals():
         if not query:
             # If no query, return all proposals
             results = get_all_proposals(category_filter, language)
-            return jsonify({
-                "query": query,
-                "results": results,
-                "total_results": len(results),
-                "category_filter": category_filter,
-                "language": language,
-                "detected_language": language,
-                "transliterated": False
-            })
         else:
-            search_result = semantic_search(query, top_k, category_filter, language)
-            return jsonify({
-                "query": query,
-                "original_query": search_result["original_query"],
-                "processed_query": search_result["processed_query"],
-                "results": search_result["results"],
-                "total_results": len(search_result["results"]),
-                "category_filter": category_filter,
-                "language": language,
-                "detected_language": search_result["detected_language"],
-                "transliterated": search_result["transliterated"]
-            })
+            results = semantic_search(query, top_k, category_filter, language)
+        
+        return jsonify({
+            "query": query,
+            "results": results,
+            "total_results": len(results),
+            "category_filter": category_filter,
+            "language": language
+        })
     
     except Exception as e:
         logger.error(f"API error: {e}")
@@ -525,28 +458,16 @@ def search_proposals_get():
         if not query:
             # If no query, return all proposals
             results = get_all_proposals(category_filter, language)
-            return jsonify({
-                "query": query,
-                "results": results,
-                "total_results": len(results),
-                "category_filter": category_filter,
-                "language": language,
-                "detected_language": language,
-                "transliterated": False
-            })
         else:
-            search_result = semantic_search(query, top_k, category_filter, language)
-            return jsonify({
-                "query": query,
-                "original_query": search_result["original_query"],
-                "processed_query": search_result["processed_query"],
-                "results": search_result["results"],
-                "total_results": len(search_result["results"]),
-                "category_filter": category_filter,
-                "language": language,
-                "detected_language": search_result["detected_language"],
-                "transliterated": search_result["transliterated"]
-            })
+            results = semantic_search(query, top_k, category_filter, language)
+        
+        return jsonify({
+            "query": query,
+            "results": results,
+            "total_results": len(results),
+            "category_filter": category_filter,
+            "language": language
+        })
     
     except Exception as e:
         logger.error(f"API error: {e}")