Update app.py

app.py

@@ -23,19 +23,31 @@ if not PINECONE_API_KEY:
 # Initialize Pinecone
 pc = Pinecone(api_key=PINECONE_API_KEY)
 # Configuration
-INDEX_NAME = "budget-proposals-embeddinggemma"  # Use EmbeddingGemma index
+# Index names for different models
+INDEX_NAME_EN = "budget-proposals-optimized"  # 384 dimensions for all-MiniLM-L6-v2 (English documents)
+INDEX_NAME_MULTILINGUAL = "budget-proposals-embeddinggemma"  # 768 dimensions for EmbeddingGemma (Sinhala/Tamil)
 
-# Load embedding model - Google's EmbeddingGemma-300m (gated model)
-# Requires HF_TOKEN secret in Hugging Face Spaces
+# Load embedding models - Hybrid approach for better performance
+# English: all-MiniLM-L6-v2 (better domain understanding)
+# Sinhala/Tamil: EmbeddingGemma-300m (better multilingual support)
 import os
 from huggingface_hub import login
 
-# Login to Hugging Face if token is available
+# Login to Hugging Face if token is available (for EmbeddingGemma)
 hf_token = os.getenv('HF_TOKEN')
 if hf_token:
     login(token=hf_token)
 
-embed_model = SentenceTransformer("google/embeddinggemma-300m")
+# Load both models
+embed_model_en = SentenceTransformer("all-MiniLM-L6-v2")
+embed_model_multilingual = SentenceTransformer("google/embeddinggemma-300m")
+
+def get_embedding_model(language):
+    """Get the appropriate embedding model based on language"""
+    if language == 'en':
+        return embed_model_en
+    else:  # si, ta, or any other language
+        return embed_model_multilingual
 
 # Load dynamic metadata
 def load_dynamic_metadata():
@@ -64,10 +76,13 @@ def get_language_specific_data(proposal_data, field, language='en'):
     
     return ''
 
-def get_pinecone_index():
-    """Get the budget proposals Pinecone index"""
+def get_pinecone_index(language='en'):
+    """Get the appropriate Pinecone index based on language"""
     try:
-        return pc.Index(INDEX_NAME)
+        if language == 'en':
+            return pc.Index(INDEX_NAME_EN)
+        else:  # si, ta, or any other language
+            return pc.Index(INDEX_NAME_MULTILINGUAL)
     except Exception as e:
         logger.error(f"Error accessing Pinecone index: {e}")
         return None
@@ -79,11 +94,13 @@ def semantic_search(query: str, top_k=1, category_filter=None, language='en'):
         global DYNAMIC_METADATA
         DYNAMIC_METADATA = load_dynamic_metadata()
         
-        pc_index = get_pinecone_index()
+        pc_index = get_pinecone_index(language)
         if not pc_index:
             return []
         
-        query_emb = embed_model.encode(query).tolist()
+        # Use language-specific embedding model
+        model = get_embedding_model(language)
+        query_emb = model.encode(query).tolist()
         
         # Build filter if category is specified
         filter_dict = {"source": "budget_proposals"}
@@ -196,7 +213,7 @@ def get_all_proposals(category_filter=None, language='en'):
         global DYNAMIC_METADATA
         DYNAMIC_METADATA = load_dynamic_metadata()
         
-        pc_index = get_pinecone_index()
+        pc_index = get_pinecone_index(language)
         if not pc_index:
             logger.warning("Pinecone index not available, returning empty list")
             return []
@@ -207,8 +224,11 @@ def get_all_proposals(category_filter=None, language='en'):
             filter_dict["category"] = category_filter
         
         # Query with a dummy vector to get all documents
-        # Use a more realistic dummy vector (all 0.1 instead of 0.0)
-        dummy_vector = [0.1] * 768  # 768 is the dimension of EmbeddingGemma-300m
+        # Use language-specific vector dimensions
+        if language == 'en':
+            dummy_vector = [0.1] * 384  # 384 is the dimension of all-MiniLM-L6-v2
+        else:  # si, ta, or any other language
+            dummy_vector = [0.1] * 768  # 768 is the dimension of EmbeddingGemma-300m
         res = pc_index.query(
             vector=dummy_vector,
             top_k=100,  # Get all proposals