add: multi-chunksize splitter for better sematic precision

qa/utils.py

@@ -137,22 +137,22 @@ def text_to_docs(pages: Union[Text, Tuple[Text]], **kwargs) -> List[HashDocument
         
         # Split pages into chunks
         doc_chunks = []
-        # Get the text splitter
-        text_splitter = get_text_splitter()
-
-        for doc in page_docs:
-            # this splits the page into chunks
-            chunks = text_splitter.split_text(doc.page_content)
-            for i, chunk in enumerate(chunks):
-                # Create a new document for each individual chunk
-                new_doc = HashDocument(
-                    page_content=chunk, 
-                    metadata={"file_name": doc.metadata["file_name"], "page": doc.metadata["page"], "chunk": i}
-                )
-                # Add sources to metadata for retrieval later on
-                new_doc.metadata["source"] = \
-                    f"{new_doc.metadata['file_name']}/Page-{new_doc.metadata['page']}/Chunk-{new_doc.metadata['chunk']}"
-                doc_chunks.append(new_doc)
+        for ntokens in [50,250,500,750]:
+            # Get the text splitter
+            text_splitter = get_text_splitter(chunk_size=ntokens, chunk_overlap=ntokens//10)
+            for doc in page_docs:
+                # this splits the page into chunks
+                chunks = text_splitter.split_text(doc.page_content)
+                for i, chunk in enumerate(chunks):
+                    # Create a new document for each individual chunk
+                    new_doc = HashDocument(
+                        page_content=chunk, 
+                        metadata={"file_name": doc.metadata["file_name"], "page": doc.metadata["page"], "chunk": i}
+                    )
+                    # Add sources to metadata for retrieval later on
+                    new_doc.metadata["source"] = \
+                        f"{new_doc.metadata['file_name']}/Page-{new_doc.metadata['page']}/Chunk-{new_doc.metadata['chunk']}/Chunksize-{ntokens}"
+                    doc_chunks.append(new_doc)
         
         return doc_chunks
 
@@ -193,8 +193,8 @@ def get_answer(
     chain = load_qa_with_sources_chain(
         ChatOpenAI(temperature=0, openai_api_key=st.session_state.get("OPENAI_API_KEY"), model=model, streaming=stream_answer), 
         chain_type="stuff",
-        prompt=STUFF_PROMPT
-        # verbose=True,
+        prompt=STUFF_PROMPT,
+        verbose=True,
         # chain_type_kwargs={
         #     "verbose": True,
         #     "prompt": query,