Update loaders/common.py

loaders/common.py

@@ -6,86 +6,100 @@ from langchain.schema import Document
 import streamlit as st
 from langchain.text_splitter import RecursiveCharacterTextSplitter
 from stats import add_usage
-import re
-
-def clean_chat_text(text):
-    """Clean chat export text to remove special characters and format consistently"""
-    # Remove non-printable characters
-    text = ''.join(char for char in text if char.isprintable())
-    
-    # Clean up WhatsApp-style timestamps and phone numbers
-    text = re.sub(r'\[\d{1,2}/\d{1,2}/\d{2,4},\s+\d{1,2}:\d{1,2}:\d{1,2}\s+[AP]M\]', '', text)
-    text = re.sub(r'‪\+\d{2,3}\s*\d{3,10}\s*\d{3,10}‬', '', text)
-    
-    # Remove joining messages
-    text = re.sub(r'joined using this group\'s invite link', '', text)
-    
-    # Remove extra whitespace
-    text = ' '.join(text.split())
-    
-    return text
 
 def process_file(vector_store, file, loader_class, file_suffix, stats_db=None):
-    documents = []
-    file_name = file.name
-    file_size = file.size
-    if st.secrets.self_hosted == "false":
-        if file_size > 1000000:
-            st.error("File size is too large. Please upload a file smaller than 1MB or self host.")
-            return
-            
-    dateshort = time.strftime("%Y%m%d")
-    with tempfile.NamedTemporaryFile(delete=False, suffix=file_suffix) as tmp_file:
-        tmp_file.write(file.getvalue())
-        tmp_file.flush()
-        loader = loader_class(tmp_file.name)
-        documents = loader.load()
-        file_sha1 = compute_sha1_from_file(tmp_file.name)
-    os.remove(tmp_file.name)
-    
-    chunk_size = st.session_state['chunk_size']
-    chunk_overlap = st.session_state['chunk_overlap']
-    text_splitter = RecursiveCharacterTextSplitter.from_tiktoken_encoder(chunk_size=chunk_size, chunk_overlap=chunk_overlap)
-    
-    documents = text_splitter.split_documents(documents)
-    
-    # Clean the text content before creating metadata
-    docs_with_metadata = [Document(page_content=clean_chat_text(doc.page_content),
-                                 metadata={"file_sha1": file_sha1,
-                                         "file_size": file_size,
-                                         "file_name": file_name, 
-                                         "chunk_size": chunk_size,
-                                         "chunk_overlap": chunk_overlap,
-                                         "date": dateshort,
-                                         "user": st.session_state["username"]}) 
-                         for doc in documents]
-    
     try:
-        # Add debug logging before vector store addition
-        print(f"Attempting to add {len(docs_with_metadata)} documents")
-        print(f"Sample cleaned content: {docs_with_metadata[0].page_content[:200] if docs_with_metadata else 'No documents'}")
+        print("=== Starting file processing ===")
+        print(f"Initial file details - Name: {file.name}, Size: {file.size}")
         
-        vector_store.add_documents(docs_with_metadata)
+        documents = []
+        file_name = file.name
+        file_size = file.size
+        if st.secrets.self_hosted == "false":
+            if file_size > 1000000:
+                st.error("File size is too large. Please upload a file smaller than 1MB or self host.")
+                return
+                
+        dateshort = time.strftime("%Y%m%d")
         
-        if stats_db:
-            add_usage(stats_db, "embedding", "file", metadata={"file_name": file_name,
-                                                             "file_type": file_suffix, 
-                                                             "chunk_size": chunk_size,
-                                                             "chunk_overlap": chunk_overlap})
-    except Exception as e:
-        print(f"Error adding documents to vector store:")
-        print(f"Exception: {str(e)}")
-        print(f"Input details:")
-        print(f"File name: {file_name}")
-        print(f"File size: {file_size}")
-        print(f"File SHA1: {file_sha1}")
-        print(f"Number of documents: {len(docs_with_metadata)}")
-        print(f"Chunk size: {chunk_size}")
-        print(f"Chunk overlap: {chunk_overlap}")
-        print(f"First document preview (truncated):")
-        if docs_with_metadata:
-            print(docs_with_metadata[0].page_content[:500])
+        # Debug loading
+        print("=== Document Loading ===")
+        with tempfile.NamedTemporaryFile(delete=False, suffix=file_suffix) as tmp_file:
+            tmp_file.write(file.getvalue())
+            tmp_file.flush()
+            print(f"Temporary file created: {tmp_file.name}")
             
-        # Additional debug info for vector store
-        print(f"Vector store type: {type(vector_store).__name__}")
+            loader = loader_class(tmp_file.name)
+            documents = loader.load()
+            print(f"Number of documents after loading: {len(documents)}")
+            print("First document content preview:")
+            if documents:
+                print(documents[0].page_content[:200])
+            
+            file_sha1 = compute_sha1_from_file(tmp_file.name)
+        os.remove(tmp_file.name)
+        
+        # Debug splitting
+        print("\n=== Document Splitting ===")
+        chunk_size = st.session_state['chunk_size']
+        chunk_overlap = st.session_state['chunk_overlap']
+        print(f"Splitting with chunk_size: {chunk_size}, overlap: {chunk_overlap}")
+        
+        text_splitter = RecursiveCharacterTextSplitter.from_tiktoken_encoder(chunk_size=chunk_size, chunk_overlap=chunk_overlap)
+        documents = text_splitter.split_documents(documents)
+        print(f"Number of documents after splitting: {len(documents)}")
+        
+        # Debug metadata creation
+        print("\n=== Creating Documents with Metadata ===")
+        docs_with_metadata = []
+        for i, doc in enumerate(documents):
+            if isinstance(doc.page_content, str):
+                if "error" in doc.page_content.lower():
+                    print(f"WARNING: Found potential error message in document {i}:")
+                    print(doc.page_content[:200])
+                    continue  # Skip this document
+                    
+                new_doc = Document(
+                    page_content=doc.page_content,
+                    metadata={
+                        "file_sha1": file_sha1,
+                        "file_size": file_size,
+                        "file_name": file_name, 
+                        "chunk_size": chunk_size,
+                        "chunk_overlap": chunk_overlap,
+                        "date": dateshort,
+                        "user": st.session_state["username"]
+                    }
+                )
+                docs_with_metadata.append(new_doc)
+            else:
+                print(f"WARNING: Document {i} has non-string content type: {type(doc.page_content)}")
+                print(f"Content: {str(doc.page_content)[:200]}")
+        
+        print(f"Final number of documents to be added: {len(docs_with_metadata)}")
+        
+        # Vector store addition
+        try:
+            vector_store.add_documents(docs_with_metadata)
+            if stats_db:
+                add_usage(stats_db, "embedding", "file", metadata={
+                    "file_name": file_name,
+                    "file_type": file_suffix, 
+                    "chunk_size": chunk_size,
+                    "chunk_overlap": chunk_overlap
+                })
+        except Exception as e:
+            print(f"\n=== Vector Store Addition Error ===")
+            print(f"Exception: {str(e)}")
+            print(f"Input details:")
+            print(f"File name: {file_name}")
+            print(f"File size: {file_size}")
+            print(f"File SHA1: {file_sha1}")
+            print(f"Number of documents: {len(docs_with_metadata)}")
+            print(f"Vector store type: {type(vector_store).__name__}")
+            raise
+
+    except Exception as e:
+        print(f"\n=== General Processing Error ===")
+        print(f"Exception occurred during file processing: {str(e)}")
         raise