Sửa lại phần in ra output box cho run_indexing

data_indexing.py

@@ -7,6 +7,11 @@ import sys
 from typing import List, Dict, Tuple, Any, Optional
 import uuid
 
+# Add project root to Python path
+project_root = os.path.abspath(os.path.join(os.path.dirname(__file__), '..', '..'))
+if project_root not in sys.path:
+    sys.path.insert(0, project_root)
+
 from PIL import Image
 from FlagEmbedding import BGEM3FlagModel
 import gradio as gr
@@ -17,7 +22,6 @@ from qdrant_client.http.models import Modifier, Distance, SparseVectorParams, Ve
 import torch
 from transformers import EfficientNetModel, AutoImageProcessor
 from pymongo import MongoClient
-import contextlib
 
 from config import (
     QDRANT_COLLECTION_NAME_SPCHIEUSANG,
@@ -147,26 +151,6 @@ mongodb_solution_collections = {
     "nha_o_xa_hoi": "gp_nha_o_xa_hoi"
 }
 
-class OutputCapture:
-    """Context manager to capture stdout and stderr"""
-    def __init__(self):
-        self.buffer = io.StringIO()
-        self.old_stdout = None
-        self.old_stderr = None
-    
-    def __enter__(self):
-        self.old_stdout = sys.stdout
-        self.old_stderr = sys.stderr
-        sys.stdout = self.buffer
-        sys.stderr = self.buffer
-        return self.buffer
-    
-    def __exit__(self, *args):
-        sys.stdout = self.old_stdout
-        sys.stderr = self.old_stderr
-    
-    def getvalue(self):
-        return self.buffer.getvalue()
 
 """=================MONGODB CONNECTION========================"""
 class MongoDBConnection:
@@ -702,72 +686,48 @@ class ProductIndexing:
             reload: Whether to recreate collections
             hybrid_mode: Whether to use hybrid text embedding (BGEM3)
         """
-        with OutputCapture() as output:
+        if reload:
             try:
-                if reload:
-                    try:
-                        for collection in product_collections:
-                            self.client.recreate_collection(
-                                collection_name=collection,
-                                vectors_config=product_vectors_config,
-                                sparse_vectors_config=sparse_vectors_config
-                            )
-                        print("✅ All product collections recreated.")
-                    except Exception as e:
-                        print(f"❌ Error while recreating collections: {e}")
-                        return output.getvalue()
-
-                # Setup MongoDB connection
-                if not self.mongodb_conn:
-                    if not self.setup_mongodb():
-                        print("❌ Failed to connect to MongoDB. Aborting indexing.")
-                        return output.getvalue()
-
-                # Create embedding processor
-                embed_object = ProductEmbedding()
-                
-                for collection, product_type in zip(product_collections, product_types):
-                    print(f"\n{'='*60}")
-                    print(f"🔄 Processing {product_type} data from MongoDB...")
-                    print(f"{'='*60}")
-                    
-                    try:
-                        # Generate embeddings for specific product type
-                        embeddings = embed_object.run_embedding(
-                            product_type=product_type,
-                            mongodb_conn=self.mongodb_conn,
-                            hybrid_mode=hybrid_mode
-                        )
-                        
-                        # Index embeddings to specific collection
-                        self.index(embeddings, collection)
-                        self._create_payload_indexes_for_product_type(product_type, collection)
-                        
-                        print(f"✅ Completed indexing for {product_type}")
-                        
-                    except Exception as e:
-                        print(f"❌ Error indexing {product_type}: {e}")
-                        import traceback
-                        print(traceback.format_exc())
-                
-                # Close MongoDB connection
-                if self.mongodb_conn:
-                    self.mongodb_conn.close()
-                    self.mongodb_conn = None
-                
-                print(f"\n{'='*60}")
-                print("🎉 All indexing completed!")
-                print(f"{'='*60}")
-                
+                for collection in product_collections:
+                    self.client.recreate_collection(
+                        collection_name=collection,
+                        vectors_config=product_vectors_config,
+                        sparse_vectors_config=sparse_vectors_config
+                    )
+                print("All product collections recreated.")
             except Exception as e:
-                print(f"❌ Fatal error during indexing: {e}")
-                import traceback
-                print(traceback.format_exc())
+                print(f"Error while recreating collections: {e}")
+                return
+
+        # Setup MongoDB connection
+        if not self.mongodb_conn:
+            if not self.setup_mongodb():
+                print("❌ Failed to connect to MongoDB. Aborting indexing.")
+                return
+
+        # Create embedding processor
+        embed_object = ProductEmbedding()
+        
+        for collection, product_type in zip(product_collections, product_types):
+            print(f"\n🔄 Processing {product_type} data from MongoDB...")
+            
+            # Generate embeddings for specific product type
+            embeddings = embed_object.run_embedding(
+                product_type=product_type,
+                mongodb_conn=self.mongodb_conn,
+                hybrid_mode=hybrid_mode
+            )
             
-            return output.getvalue()
+            # Index embeddings to specific collection
+            self.index(embeddings, collection)
+            self._create_payload_indexes_for_product_type(product_type, collection)
+        
+        # Close MongoDB connection
+        if self.mongodb_conn:
+            self.mongodb_conn.close()
+            self.mongodb_conn = None
     
-    def indexing_single_product_type(self, product_type: str, collection_name: str, 
-                                    hybrid_mode: bool = True) -> str:
+    def indexing_single_product_type(self, product_type: str, collection_name: str, hybrid_mode: bool = True) -> str:
         """
         Indexing a single product group into its Qdrant collection from MongoDB
         Args:
@@ -775,56 +735,46 @@ class ProductIndexing:
             collection_name: Qdrant collection name
             hybrid_mode: Whether to use hybrid text embedding (BGEM3)
         """
-        with OutputCapture() as output:
-            try:
-                print(f"{'='*60}")
-                print(f"🚀 Starting indexing for {product_type}")
-                print(f"{'='*60}\n")
-                
-                self.client.recreate_collection(
-                    collection_name=collection_name,
-                    vectors_config=product_vectors_config,
-                    sparse_vectors_config=sparse_vectors_config
-                )
-                print(f"✅ Collection {collection_name} created\n")
-
-                # Setup MongoDB connection
-                if not self.mongodb_conn:
-                    if not self.setup_mongodb():
-                        print("❌ Failed to connect to MongoDB")
-                        return output.getvalue()
-
-                # Create embedding processor
-                embed_object = ProductEmbedding()
-
-                print(f"🔄 Processing {product_type} data from MongoDB...")
-                embeddings = embed_object.run_embedding(
-                    product_type=product_type,
-                    mongodb_conn=self.mongodb_conn,
-                    hybrid_mode=hybrid_mode
-                )
-                
-                print(f"\n📊 Indexing to Qdrant...")
-                self.index(embeddings, collection_name)
-                
-                print(f"\n🔍 Creating payload indexes...")
-                self._create_payload_indexes_for_product_type(product_type, collection_name)
+        buffer = io.StringIO()
+        sys.stdout = buffer
 
-                # Close MongoDB connection
-                if self.mongodb_conn:
-                    self.mongodb_conn.close()
-                    self.mongodb_conn = None
-                
-                print(f"\n{'='*60}")
-                print(f"✅ Successfully completed indexing for {product_type}")
-                print(f"{'='*60}")
+        try:
+            self.client.recreate_collection(
+                collection_name=collection_name,
+                vectors_config=product_vectors_config,
+                sparse_vectors_config=sparse_vectors_config
+            )
+            print(f"Collection {collection_name} created")
+
+            # Setup MongoDB connection
+            if not self.mongodb_conn:
+                if not self.setup_mongodb():
+                    print("❌ Failed to connect to MongoDB")
+                    sys.stdout = sys.__stdout__
+                    return buffer.getvalue()
+
+            # Create embedding processor
+            embed_object = ProductEmbedding()
+
+            print(f"\n🔄 Processing {product_type} data from MongoDB...")
+            embeddings = embed_object.run_embedding(
+                product_type=product_type,
+                mongodb_conn=self.mongodb_conn,
+                hybrid_mode=hybrid_mode
+            )
+            self.index(embeddings, collection_name)
 
-            except Exception as e:
-                print(f"❌ Error while indexing product type {product_type}: {e}")
-                import traceback
-                print(traceback.format_exc())
-            
-            return output.getvalue()
+            # Close MongoDB connection
+            if self.mongodb_conn:
+                self.mongodb_conn.close()
+                self.mongodb_conn = None
+
+        except Exception as e:
+            print(f"Error while indexing product type {product_type}: {e}")
+        
+        self._create_payload_indexes_for_product_type(product_type, collection_name)
+        sys.stdout = sys.__stdout__
+        return buffer.getvalue()
 
     def _create_payload_indexes_for_product_type(self, product_type: str, collection_name: str):
         """Create payload indexes based on product type field schemas"""
@@ -1085,109 +1035,79 @@ class SolutionIndexing:
 
     def run_indexing(self, reload: bool = True):
         """Index all solution data from MongoDB into Qdrant collections."""
-        with OutputCapture() as output:
+        if reload:
             try:
-                if reload:
-                    try:
-                        for collection in solution_collections:
-                            self.client.recreate_collection(
-                                collection_name=collection,
-                                vectors_config=qdrant_client.http.models.VectorParams(
-                                    size=768,
-                                    distance=qdrant_client.http.models.Distance.COSINE,
-                                )
-                            )
-                        print("✅ All solution collections recreated.")
-                    except Exception as e:
-                        print(f"❌ Error while recreating collections: {e}")
-                        return output.getvalue()
-
-                # Setup MongoDB connection
-                if not self.mongodb_conn:
-                    if not self.setup_mongodb():
-                        print("❌ Failed to connect to MongoDB. Aborting indexing.")
-                        return output.getvalue()
-
-                # Create embedding processor
-                embed_object = SolutionEmbedding()
-                
-                for collection, solution_type in zip(solution_collections, solution_types):
-                    print(f"\n{'='*60}")
-                    print(f"🔄 Processing {solution_type} data from MongoDB...")
-                    print(f"{'='*60}")
-                    
-                    try:
-                        embeddings = embed_object.run_embedding(solution_type, self.mongodb_conn)
-                        self.index(embeddings, collection)
-                        print(f"✅ Completed indexing for {solution_type}")
-                    except Exception as e:
-                        print(f"❌ Error indexing {solution_type}: {e}")
-                        import traceback
-                        print(traceback.format_exc())
-                
-                # Close MongoDB connection
-                if self.mongodb_conn:
-                    self.mongodb_conn.close()
-                    self.mongodb_conn = None
-                
-                print(f"\n{'='*60}")
-                print("🎉 All solution indexing completed!")
-                print(f"{'='*60}")
-                
+                for collection in solution_collections:
+                    self.client.recreate_collection(
+                        collection_name=collection,
+                        vectors_config=qdrant_client.http.models.VectorParams(
+                            size=768,
+                            distance=qdrant_client.http.models.Distance.COSINE,
+                        )
+                    )
+                print("All solution collections recreated.")
             except Exception as e:
-                print(f"❌ Fatal error during indexing: {e}")
-                import traceback
-                print(traceback.format_exc())
-            
-            return output.getvalue()
+                print(f"Error while recreating collections: {e}")
+                return
+
+        # Setup MongoDB connection
+        if not self.mongodb_conn:
+            if not self.setup_mongodb():
+                print("❌ Failed to connect to MongoDB. Aborting indexing.")
+                return
+
+        # Create embedding processor
+        embed_object = SolutionEmbedding()
+        
+        for collection, solution_type in zip(solution_collections, solution_types):
+            print(f"\n🔄 Processing {solution_type} data from MongoDB...")
+            embeddings = embed_object.run_embedding(solution_type, self.mongodb_conn)
+            self.index(embeddings, collection)
+        
+        # Close MongoDB connection
+        if self.mongodb_conn:
+            self.mongodb_conn.close()
+            self.mongodb_conn = None
     
     def indexing_single_solution(self, solution: str, collection_name: str) -> str:
         """Indexing a single solution into its Qdrant collection from MongoDB"""
-        with OutputCapture() as output:
-            try:
-                print(f"{'='*60}")
-                print(f"🚀 Starting indexing for {solution}")
-                print(f"{'='*60}\n")
-                
-                self.client.recreate_collection(
-                    collection_name=collection_name,
-                    vectors_config=qdrant_client.http.models.VectorParams(
-                        size=768,
-                        distance=qdrant_client.http.models.Distance.COSINE,
-                    )
+        buffer = io.StringIO()
+        sys.stdout = buffer
+
+        try:
+            self.client.recreate_collection(
+                collection_name=collection_name,
+                vectors_config=qdrant_client.http.models.VectorParams(
+                    size=768,
+                    distance=qdrant_client.http.models.Distance.COSINE,
                 )
-                print(f"✅ Collection {collection_name} created\n")
+            )
+            print(f"Collection {collection_name} created")
 
-                # Setup MongoDB connection
-                if not self.mongodb_conn:
-                    if not self.setup_mongodb():
-                        print("❌ Failed to connect to MongoDB")
-                        return output.getvalue()
+            # Setup MongoDB connection
+            if not self.mongodb_conn:
+                if not self.setup_mongodb():
+                    print("❌ Failed to connect to MongoDB")
+                    sys.stdout = sys.__stdout__
+                    return buffer.getvalue()
 
-                # Create embedding processor
-                embed_object = SolutionEmbedding()
+            # Create embedding processor
+            embed_object = SolutionEmbedding()
 
-                print(f"🔄 Processing {solution} data from MongoDB...")
-                embeddings = embed_object.run_embedding(solution, self.mongodb_conn)
-                
-                print(f"\n📊 Indexing to Qdrant...")
-                self.index(embeddings, collection_name)
-                
-                # Close MongoDB connection
-                if self.mongodb_conn:
-                    self.mongodb_conn.close()
-                    self.mongodb_conn = None
-                
-                print(f"\n{'='*60}")
-                print(f"✅ Successfully completed indexing for {solution}")
-                print(f"{'='*60}")
-                    
-            except Exception as e:
-                print(f"❌ Error while indexing solution {solution}: {e}")
-                import traceback
-                print(traceback.format_exc())
+            print(f"\n🔄 Processing {solution} data from MongoDB...")
+            embeddings = embed_object.run_embedding(solution, self.mongodb_conn)
+            self.index(embeddings, collection_name)
             
-            return output.getvalue()
+            # Close MongoDB connection
+            if self.mongodb_conn:
+                self.mongodb_conn.close()
+                self.mongodb_conn = None
+                
+        except Exception as e:
+            print(f"Error while recreating collection and indexing solution {solution}: {e}")
+        
+        sys.stdout = sys.__stdout__
+        return buffer.getvalue()
 
 
 """=================GRADIO UI========================"""
@@ -1200,13 +1120,7 @@ def create_indexing_interface():
         gr.Markdown("# 🗄️ Qdrant Data Indexing System (MongoDB)")
         gr.Markdown("Recreate Qdrant Collections and Index Data from MongoDB Atlas")
         
-        output_box = gr.Textbox(
-            lines=20, 
-            label="📋 Logs", 
-            interactive=False,
-            show_copy_button=True,
-            max_lines=30
-        )
+        output_box = gr.Textbox(lines=15, label="📋 Logs", interactive=False)
         
         gr.Markdown("---")
         gr.Markdown("## 🏢 Giải pháp (Solutions)")
@@ -1264,9 +1178,15 @@ def create_indexing_interface():
                 inputs=[gr.State("nha_o_xa_hoi"), gr.State(QDRANT_COLLECTION_NAME_GPNOXH)],
                 outputs=output_box)
             
+            def index_all_solutions():
+                buffer = io.StringIO()
+                sys.stdout = buffer
+                solution_indexing.run_indexing(reload=True)
+                sys.stdout = sys.__stdout__
+                return buffer.getvalue()
+            
             gr.Button("✨ Tất cả GP", variant="primary").click(
-                solution_indexing.run_indexing,
-                inputs=gr.State(True),
+                index_all_solutions,
                 outputs=output_box)
         
         gr.Markdown("---")
@@ -1309,13 +1229,20 @@ def create_indexing_interface():
             inputs=[gr.State("thiet_bi_dien"), gr.State(QDRANT_COLLECTION_NAME_SPTHIETBIDIEN), gr.State(True)],
             outputs=output_box)
         
+        def index_all_products():
+            buffer = io.StringIO()
+            sys.stdout = buffer
+            product_indexing.run_indexing(reload=True, hybrid_mode=True)
+            sys.stdout = sys.__stdout__
+            return buffer.getvalue()
+        
         btn_all_products.click(
-            product_indexing.run_indexing,
-            inputs=[gr.State(True), gr.State(True)],
+            index_all_products,
             outputs=output_box)
     
     return demo
 
+
 if __name__ == "__main__":
     demo = create_indexing_interface()
     demo.launch()