modifying path to align with current file structure. folder structure needs rework

src/pdf-version/data/loaders.py

@@ -5,7 +5,7 @@ import os
 from typing import List, Dict
 
 
-def load_annotations(file_path: str = 'mapping.json') -> List[Dict]:
+def load_annotations(file_path: str = None) -> List[Dict]:
     """Load medical annotations from JSON file.
     
     Args:
@@ -14,6 +14,12 @@ def load_annotations(file_path: str = 'mapping.json') -> List[Dict]:
     Returns:
         List of annotation dictionaries.
     """
+    if file_path is None:
+        # Get project root directory (3 levels up from this file)
+        from pathlib import Path
+        root_dir = Path(__file__).parent.parent.parent.parent
+        file_path = root_dir / 'embeddings' / 'mapping.json'
+    
     try:
         with open(file_path, 'r', encoding='utf-8') as f:
             annotations = json.load(f)
@@ -25,7 +31,7 @@ def load_annotations(file_path: str = 'mapping.json') -> List[Dict]:
         return []
 
 
-def filter_pdf_files(annotations: List[Dict], assets_dir: str = "assets") -> List[str]:
+def filter_pdf_files(annotations: List[Dict], assets_dir: str = None) -> List[str]:
     """Filter and validate PDF files from annotations.
     
     Args:
@@ -35,6 +41,12 @@ def filter_pdf_files(annotations: List[Dict], assets_dir: str = "assets") -> Lis
     Returns:
         List of valid PDF filenames.
     """
+    if assets_dir is None:
+        # Get project root directory
+        from pathlib import Path
+        root_dir = Path(__file__).parent.parent.parent.parent
+        assets_dir = root_dir / 'assets'
+    
     pdf_files = []
 
     for item in annotations:

src/pdf-version/demos/demo_runner.py

@@ -2,13 +2,13 @@
 
 from typing import Optional
 
-from src.models.embedding_models import load_biomedbert_model
-from src.data.loaders import load_annotations
-from src.indexing.document_indexer import build_document_index
-from src.indexing.embedding_creator import create_tag_embeddings, create_chunk_embeddings
-from src.indexing.storage import save_document_system, load_document_system
-from src.retrieval.document_retriever import create_document_tag_mapping, find_relevant_documents
-from src.retrieval.chunk_retriever import find_relevant_chunks, get_documents_for_rag, get_chunks_for_rag
+from models.embedding_models import load_biomedbert_model
+from data.loaders import load_annotations
+from indexing.document_indexer import build_document_index
+from indexing.embedding_creator import create_tag_embeddings, create_chunk_embeddings
+from indexing.storage import save_document_system, load_document_system
+from retrieval.document_retriever import create_document_tag_mapping, find_relevant_documents
+from retrieval.chunk_retriever import find_relevant_chunks, get_documents_for_rag, get_chunks_for_rag
 
 
 def build_medical_rag_system(enable_chunk_embeddings: bool = True):

src/pdf-version/generate_embeddings.py

@@ -6,10 +6,10 @@ Quick script to generate new embeddings with sentence-based chunking
 import sys
 from pathlib import Path
 
-# Add src directory to Python path
-sys.path.insert(0, str(Path(__file__).parent / "src"))
+# Add pdf-version directory to Python path
+sys.path.insert(0, str(Path(__file__).parent))
 
-from src.demos.demo_runner import build_medical_rag_system
+from demos.demo_runner import build_medical_rag_system
 
 def main():
     print("🚀 Starting to build medical RAG system with new sentence-based chunking...")

src/pdf-version/indexing/document_indexer.py

@@ -4,7 +4,7 @@ import os
 from typing import List, Dict
 from llama_index.core import Document
 from llama_index.core.node_parser import SentenceSplitter
-from src.data.pdf_processing import extract_pdf_content_enhanced
+from data.pdf_processing import extract_pdf_content_enhanced
 
 
 def split_text_into_chunks(text: str, chunk_size: int = 256, chunk_overlap: int = 25) -> List[Dict]:

src/pdf-version/indexing/storage.py

@@ -8,7 +8,7 @@ import numpy as np
 
 def save_document_system(document_index: Dict, tag_embeddings: Dict, 
                         doc_tag_mapping: Dict, chunk_embeddings: Dict = None, 
-                        output_dir: str = "."):
+                        output_dir: str = None):
     """Save the complete document indexing system.
     
     Args:
@@ -19,6 +19,15 @@ def save_document_system(document_index: Dict, tag_embeddings: Dict,
         output_dir: Output directory for saved files.
     """
     
+    if output_dir is None:
+        # Get project root directory
+        from pathlib import Path
+        root_dir = Path(__file__).parent.parent.parent.parent
+        output_dir = root_dir / 'embeddings' / 'pdfembeddings'
+    
+    # Ensure output directory exists
+    os.makedirs(output_dir, exist_ok=True)
+    
     # Save document index (content + metadata + chunks)
     doc_index_serializable = {}
     for doc_name, doc_info in document_index.items():
@@ -79,7 +88,7 @@ def save_document_system(document_index: Dict, tag_embeddings: Dict,
     print("✅ Document system saved to files")
 
 
-def load_document_system(input_dir: str = ".") -> Tuple[Optional[Dict], Optional[Dict], Optional[Dict], Optional[Dict]]:
+def load_document_system(input_dir: str = None) -> Tuple[Optional[Dict], Optional[Dict], Optional[Dict], Optional[Dict]]:
     """Load the complete document indexing system.
     
     Args:
@@ -89,6 +98,12 @@ def load_document_system(input_dir: str = ".") -> Tuple[Optional[Dict], Optional
         Tuple of (document_index, tag_embeddings, doc_tag_mapping, chunk_embeddings).
         Returns (None, None, None, None) if loading fails.
     """
+    if input_dir is None:
+        # Get project root directory
+        from pathlib import Path
+        root_dir = Path(__file__).parent.parent.parent.parent
+        input_dir = root_dir / 'embeddings' / 'pdfembeddings'
+    
     try:
         # Load document index
         with open(os.path.join(input_dir, 'document_index.json'), 'r', encoding='utf-8') as f:

src/pdf-version/main.py

@@ -7,10 +7,10 @@ Main entry point for the medical RAG system.
 import sys
 from pathlib import Path
 
-# Add src directory to Python path
-sys.path.insert(0, str(Path(__file__).parent / "src"))
+# Add pdf-version directory to Python path
+sys.path.insert(0, str(Path(__file__).parent))
 
-from src.demos.demo_runner import build_medical_rag_system, demo_rag_query, demo_all_strategies
+from demos.demo_runner import build_medical_rag_system, demo_rag_query, demo_all_strategies
 
 
 def main():

src/pdf-version/oncall_ai.py

@@ -8,26 +8,26 @@ Import everything from the new modular structure.
 import sys
 from pathlib import Path
 
-# Add src directory to Python path
-sys.path.insert(0, str(Path(__file__).parent / "src"))
+# Add pdf-version directory to Python path
+sys.path.insert(0, str(Path(__file__).parent))
 
 # Import all functions for backward compatibility
-from src.models.embedding_models import load_biomedbert_model, load_meditron_model
-from src.data.loaders import load_annotations, filter_pdf_files
-from src.data.pdf_processing import (
+from models.embedding_models import load_biomedbert_model, load_meditron_model
+from data.loaders import load_annotations, filter_pdf_files
+from data.pdf_processing import (
     extract_pdf_text, extract_tables_from_pdf,
     extract_images_ocr_from_pdf, extract_pdf_content_enhanced
 )
-from src.indexing.document_indexer import build_document_index, split_text_into_chunks
-from src.indexing.embedding_creator import create_text_embedding, create_tag_embeddings, create_chunk_embeddings
-from src.indexing.storage import save_document_system, load_document_system
-from src.retrieval.document_retriever import (
+from indexing.document_indexer import build_document_index, split_text_into_chunks
+from indexing.embedding_creator import create_text_embedding, create_tag_embeddings, create_chunk_embeddings
+from indexing.storage import save_document_system, load_document_system
+from retrieval.document_retriever import (
     find_relevant_documents_top_k, find_relevant_documents_top_p,
     find_relevant_documents_threshold, find_relevant_documents,
     create_document_tag_mapping
 )
-from src.retrieval.chunk_retriever import find_relevant_chunks, get_documents_for_rag, get_chunks_for_rag
-from src.demos.demo_runner import build_medical_rag_system, demo_rag_query, demo_all_strategies
+from retrieval.chunk_retriever import find_relevant_chunks, get_documents_for_rag, get_chunks_for_rag
+from demos.demo_runner import build_medical_rag_system, demo_rag_query, demo_all_strategies
 
 # Main function for backward compatibility
 def main():

src/pdf-version/rag/medical_rag_pipeline.py

@@ -7,10 +7,6 @@ from typing import Dict, List, Optional, Tuple
 from sentence_transformers import SentenceTransformer
 
 # Import existing retrieval components
-import sys
-import os
-sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
-
 from retrieval.document_retriever import find_relevant_documents
 from retrieval.chunk_retriever import find_relevant_chunks, get_chunks_for_rag
 from models.embedding_models import load_biomedbert_model
@@ -391,10 +387,10 @@ def answer_medical_query(query: str,
     return complete_result
 
 
-def load_rag_data(tag_embeddings_path: str = "tag_embeddings.json",
-                  chunk_embeddings_path: str = "chunk_embeddings.json", 
-                  doc_tag_mapping_path: str = "document_tag_mapping.json",
-                  document_index_path: str = "document_index.json") -> Tuple[SentenceTransformer, Dict, Dict, Dict, Dict]:
+def load_rag_data(tag_embeddings_path: str = None,
+                  chunk_embeddings_path: str = None, 
+                  doc_tag_mapping_path: str = None,
+                  document_index_path: str = None) -> Tuple[SentenceTransformer, Dict, Dict, Dict, Dict]:
     """
     Load all RAG data needed for medical question answering.
     
@@ -409,6 +405,28 @@ def load_rag_data(tag_embeddings_path: str = "tag_embeddings.json",
     """
     print("🔄 Loading Medical RAG Data...")
     
+    # Set default paths if not provided
+    if tag_embeddings_path is None:
+        from pathlib import Path
+        root_dir = Path(__file__).parent.parent.parent.parent
+        embeddings_dir = root_dir / 'embeddings' / 'pdfembeddings'
+        tag_embeddings_path = embeddings_dir / 'tag_embeddings.json'
+    if chunk_embeddings_path is None:
+        from pathlib import Path
+        root_dir = Path(__file__).parent.parent.parent.parent
+        embeddings_dir = root_dir / 'embeddings' / 'pdfembeddings'
+        chunk_embeddings_path = embeddings_dir / 'chunk_embeddings.json'
+    if doc_tag_mapping_path is None:
+        from pathlib import Path
+        root_dir = Path(__file__).parent.parent.parent.parent
+        embeddings_dir = root_dir / 'embeddings' / 'pdfembeddings'
+        doc_tag_mapping_path = embeddings_dir / 'document_tag_mapping.json'
+    if document_index_path is None:
+        from pathlib import Path
+        root_dir = Path(__file__).parent.parent.parent.parent
+        embeddings_dir = root_dir / 'embeddings' / 'pdfembeddings'
+        document_index_path = embeddings_dir / 'document_index.json'
+    
     # Load embedding model
     print("📦 Loading BGE Large Medical embedding model...")
     embedding_model = load_biomedbert_model()