Update main_api.py

main_api.py

@@ -5,7 +5,13 @@ import random
 import json
 import numpy as np
 import uvicorn
-import fitz  # PyMuPDF
+
+# FIX: Updated PyMuPDF import for compatibility
+try:
+    import pymupdf as fitz  # PyMuPDF >= 1.24.0 (recommended)
+except ImportError:
+    import fitz  # PyMuPDF < 1.24.0 (fallback)
+
 import pymupdf4llm
 import faiss
 from pathlib import Path
@@ -21,8 +27,6 @@ from supabase import create_client, Client
 from groq import Groq
 from sentence_transformers import SentenceTransformer
 from langchain_text_splitters import RecursiveCharacterTextSplitter, MarkdownTextSplitter
-
-
 import pickle
 
 # ==================== CONFIGURATION FOR HUGGING FACE SPACES ====================
@@ -272,7 +276,7 @@ def query(payload):
 def process_with_groq(query: str, context: str) -> str:
     """Process query with Groq LLM"""
     if not groq_client:
-        return "Groq API not configured. Please set GROQ_API_KEY environment variable."
+        return "Groq API not configured. Please set groq_token environment variable."
     
     try:
         messages = [
@@ -390,6 +394,8 @@ async def upload_document(
         if not text_content.strip():
             raise HTTPException(status_code=400, detail="No text content extracted")
         
+        logger.info(f"Extracted {len(text_content)} characters from {file.filename}")
+        
         # Optional: Upload to Supabase
         storage_filename = f"{int(time.time())}_{file.filename}"
         if supabase:
@@ -398,13 +404,14 @@ async def upload_document(
                     supabase.storage.from_(bucket_name).upload(
                         path=storage_filename,
                         file=f.read(),
-                        file_options={"content-type": "application/octet-stream"}  # Generic type
+                        file_options={"content-type": "application/octet-stream"}
                     )
             except:
                 pass  # Continue even if Supabase upload fails
         
         # Chunk document
         chunks = chunk_document(text_content, file_type)
+        logger.info(f"Created {len(chunks)} chunks for collection '{collection_name}'")
         
         # Create metadata
         file_id = str(int(time.time()))
@@ -424,6 +431,8 @@ async def upload_document(
             collection_name, chunks, metadata
         )
         
+        logger.info(f"Successfully added {chunks_created} chunks to collection '{collection_name}'")
+        
         # Clean up temp file
         try:
             os.remove(temp_file_path)
@@ -444,7 +453,6 @@ async def upload_document(
         logger.exception("Error in upload_document")
         raise HTTPException(status_code=500, detail=f"Error: {str(e)}")
 
-
 @app.post("/upload_multiple_documents")
 async def upload_multiple_documents(
     files: List[UploadFile] = File(...),
@@ -470,16 +478,20 @@ async def upload_multiple_documents(
 
 @app.post("/query_documents")
 async def query_documents(request: RAGQueryRequest):
-    """Query documents using RAG"""
+    """Query documents using RAG - FIXED VERSION"""
     store_data = vector_store_manager.get_store(request.collection_name)
     
     if not store_data:
         raise HTTPException(
             status_code=404,
-            detail=f"Collection '{request.collection_name}' not found"
+            detail=f"Collection '{request.collection_name}' not found. Please upload documents first."
         )
     
     try:
+        # Log query details
+        logger.info(f"Querying collection '{request.collection_name}' with query: '{request.query}'")
+        logger.info(f"Collection has {len(store_data['chunks'])} chunks")
+        
         # Generate query embedding
         query_embedding = embedding_model.encode([request.query])
         query_embedding = np.array(query_embedding).astype('float32')
@@ -490,25 +502,26 @@ async def query_documents(request: RAGQueryRequest):
             min(request.top_k, len(store_data['chunks']))
         )
         
-        # Check relevance threshold
-        if distances[0][0] > 1.5:
-            return {
-                "answer": "I couldn't find this information in the provided documents.",
-                "sources": [],
-                "query": request.query,
-                "collection": request.collection_name
-            }
+        # Log search results
+        logger.info(f"Search results - distances: {distances[0]}, indices: {indices[0]}")
+        
+        # FIX: Removed strict threshold - always return results
+        # The threshold was too strict and preventing valid results
         
         # Get relevant chunks
         retrieved_chunks = [store_data['chunks'][i] for i in indices[0]]
         retrieved_metadata = [store_data['metadata'][i] for i in indices[0]]
         
+        logger.info(f"Retrieved {len(retrieved_chunks)} chunks for query")
+        
         # Create context
         context_text = "\n\n".join([
             f"[Source {i+1} - {meta['filename']}]:\n{chunk}"
             for i, (chunk, meta) in enumerate(zip(retrieved_chunks, retrieved_metadata))
         ])
         
+        logger.info(f"Context length: {len(context_text)} characters")
+        
         # Generate answer
         answer = process_with_groq(request.query, context_text)
         
@@ -535,6 +548,23 @@ async def query_documents(request: RAGQueryRequest):
         logger.exception("Error in query_documents")
         raise HTTPException(status_code=500, detail=f"Query failed: {str(e)}")
 
+@app.get("/debug_collection/{collection_name}")
+async def debug_collection(collection_name: str):
+    """Debug endpoint to inspect collection contents"""
+    store_data = vector_store_manager.get_store(collection_name)
+    
+    if not store_data:
+        return {"error": f"Collection '{collection_name}' not found"}
+    
+    return {
+        "collection_name": collection_name,
+        "total_chunks": len(store_data['chunks']),
+        "dimension": store_data['dimension'],
+        "sample_chunks": store_data['chunks'][:3] if len(store_data['chunks']) > 0 else [],
+        "sample_metadata": store_data['metadata'][:3] if len(store_data['metadata']) > 0 else [],
+        "all_filenames": list(set([meta['filename'] for meta in store_data['metadata']]))
+    }
+
 @app.get("/list_collections")
 async def list_collections():
     """List all collections"""