Jeff's progress w/o embedding files

src/pdf-version/__init__.py

@@ -0,0 +1,7 @@
+"""OnCall AI - Medical RAG System
+
+A sophisticated two-tier retrieval system for emergency department medical assistance.
+"""
+
+__version__ = "1.0.0"
+__author__ = "OnCall AI Team"

src/pdf-version/data/__init__.py

@@ -0,0 +1,15 @@
+"""Data loading and PDF processing."""
+
+from .loaders import load_annotations, filter_pdf_files
+from .pdf_processing import (
+    extract_pdf_text, 
+    extract_tables_from_pdf,
+    extract_images_ocr_from_pdf,
+    extract_pdf_content_enhanced
+)
+
+__all__ = [
+    'load_annotations', 'filter_pdf_files',
+    'extract_pdf_text', 'extract_tables_from_pdf', 
+    'extract_images_ocr_from_pdf', 'extract_pdf_content_enhanced'
+]

src/pdf-version/data/loaders.py

@@ -0,0 +1,49 @@
+"""Data loading and annotation handling."""
+
+import json
+import os
+from typing import List, Dict
+
+
+def load_annotations(file_path: str = 'mapping.json') -> List[Dict]:
+    """Load medical annotations from JSON file.
+    
+    Args:
+        file_path: Path to the annotations JSON file.
+        
+    Returns:
+        List of annotation dictionaries.
+    """
+    try:
+        with open(file_path, 'r', encoding='utf-8') as f:
+            annotations = json.load(f)
+        print(f"Loaded #{len(annotations)} annotated data")
+        
+        return annotations
+    except:
+        print(f"failed to find file: {file_path}")
+        return []
+
+
+def filter_pdf_files(annotations: List[Dict], assets_dir: str = "assets") -> List[str]:
+    """Filter and validate PDF files from annotations.
+    
+    Args:
+        annotations: List of annotation dictionaries.
+        assets_dir: Directory containing PDF files.
+        
+    Returns:
+        List of valid PDF filenames.
+    """
+    pdf_files = []
+
+    for item in annotations:
+        filename = item['pdf']
+        filepath = os.path.join(assets_dir, filename)
+
+        if filename.endswith('.pdf') and os.path.exists(filepath):
+            pdf_files.append(filename)
+        else:
+            print(f"⚠️ Skipping non-pdf and non-existing files: {filename}")
+
+    return pdf_files

src/pdf-version/data/pdf_processing.py

@@ -0,0 +1,150 @@
+"""PDF content extraction and processing."""
+
+import os
+import io
+from typing import List
+import numpy as np
+import pandas as pd
+
+# PDF processing imports
+import pdfplumber
+import fitz  # PyMuPDF
+import easyocr
+from PIL import Image
+
+# LlamaIndex imports
+from llama_index.core import Document, SimpleDirectoryReader
+
+
+def extract_pdf_text(pdf_path: str) -> str:
+    """Extract plain text from PDF file.
+    
+    Args:
+        pdf_path: Path to the PDF file.
+        
+    Returns:
+        Extracted text content.
+    """
+    text_content = ""
+    try:
+        with pdfplumber.open(pdf_path) as pdf:
+            for page in pdf.pages:
+                page_text = page.extract_text()
+                if page_text:
+                    text_content += page_text + "\n"
+        return text_content
+    except Exception as e:
+        print(f"❌ PDF text extraction error {pdf_path}: {e}")
+        return ""
+
+
+def extract_tables_from_pdf(pdf_path: str) -> Document:
+    """Extract tables from PDF and convert to markdown format.
+    
+    Args:
+        pdf_path: Path to the PDF file.
+        
+    Returns:
+        Document containing extracted table content.
+    """
+    try:
+        with pdfplumber.open(pdf_path) as pdf:
+            all_tables = []
+            for page_num, page in enumerate(pdf.pages):
+                tables = page.extract_tables()
+                for table_num, table in enumerate(tables):
+                    if table:
+                        # Convert to DataFrame then markdown
+                        df = pd.DataFrame(table[1:], columns=table[0])
+                        table_text = f"Page{page_num+1}Table{table_num+1}:\n{df.to_markdown(index=False)}"
+                        all_tables.append(table_text)
+
+            return Document(text="\n\n".join(all_tables))
+    except Exception as e:
+        print(f"⚠️ pdfplumber table extraction failed: {e}")
+        return Document(text="")
+
+
+def extract_images_ocr_from_pdf(pdf_path: str) -> Document:
+    """Extract text from images in PDF using OCR.
+    
+    Args:
+        pdf_path: Path to the PDF file.
+        
+    Returns:
+        Document containing OCR-extracted text.
+    """
+    try:
+        ocr_reader = easyocr.Reader(['en'], gpu=False)
+        doc = fitz.open(pdf_path)
+
+        image_texts = []
+        total_images = 0
+
+        for page_num, page in enumerate(doc):
+            images = page.get_images(full=True)
+            total_images += len(images)
+
+            for img_index, img in enumerate(images):
+                try:
+                    xref = img[0]
+                    base_image = doc.extract_image(xref)
+                    image_bytes = base_image["image"]
+
+                    # Convert to PIL image and perform OCR
+                    image_pil = Image.open(io.BytesIO(image_bytes)).convert("RGB")
+                    ocr_result = ocr_reader.readtext(np.array(image_pil), detail=0)
+                    ocr_text = "\n".join(ocr_result).strip()
+
+                    if ocr_text:
+                        image_texts.append(f"Page {page_num+1} image {img_index+1}:\n{ocr_text}")
+
+                except Exception as e:
+                    continue
+
+        doc.close()
+
+        all_ocr_text = "\n\n".join(image_texts)
+        if image_texts:
+            print(f"✅ Extracted text from {len(image_texts)}/{total_images} images")
+
+        return Document(text=all_ocr_text)
+
+    except Exception as e:
+        print(f"⚠️ Image OCR extraction failed {pdf_path}: {e}")
+        return Document(text="")
+
+
+def extract_pdf_content_enhanced(pdf_path: str) -> List[Document]:
+    """Enhanced PDF content extraction combining text, tables, and OCR.
+    
+    Args:
+        pdf_path: Path to the PDF file.
+        
+    Returns:
+        List of Document objects containing extracted content.
+    """
+    documents = []
+
+    print(f"🔄 Processing PDF: {os.path.basename(pdf_path)}")
+
+    # 1. Basic text extraction
+    try:
+        text_docs = SimpleDirectoryReader(input_files=[pdf_path]).load_data()
+        documents.extend(text_docs)
+        print(f"✅ Extracted basic text content")
+    except Exception as e:
+        print(f"❌ Basic text extraction failed: {e}")
+
+    # 2. Table extraction
+    table_doc = extract_tables_from_pdf(pdf_path)
+    if table_doc.text.strip():
+        documents.append(table_doc)
+
+    # 3. Image OCR extraction
+    ocr_doc = extract_images_ocr_from_pdf(pdf_path)
+    if ocr_doc.text.strip():
+        documents.append(ocr_doc)
+
+    print(f"✅ Created {len(documents)} document objects in total")
+    return documents

src/pdf-version/demos/__init__.py

@@ -0,0 +1,9 @@
+"""Demo and testing functions."""
+
+from .demo_runner import (
+    build_medical_rag_system,
+    demo_rag_query,
+    demo_all_strategies
+)
+
+__all__ = ['build_medical_rag_system', 'demo_rag_query', 'demo_all_strategies']

src/pdf-version/demos/demo_runner.py

@@ -0,0 +1,138 @@
+"""Demo and testing functionality."""
+
+from typing import Optional
+
+from src.models.embedding_models import load_biomedbert_model
+from src.data.loaders import load_annotations
+from src.indexing.document_indexer import build_document_index
+from src.indexing.embedding_creator import create_tag_embeddings, create_chunk_embeddings
+from src.indexing.storage import save_document_system, load_document_system
+from src.retrieval.document_retriever import create_document_tag_mapping, find_relevant_documents
+from src.retrieval.chunk_retriever import find_relevant_chunks, get_documents_for_rag, get_chunks_for_rag
+
+
+def build_medical_rag_system(enable_chunk_embeddings: bool = True):
+    """Build the complete medical RAG system with document-tag indexing."""
+    print("🏥 OnCall AI - Medical RAG System Starting")
+    print("=" * 60)
+
+    # Load model and data
+    embedding_model = load_biomedbert_model()
+    annotations = load_annotations()
+
+    if not annotations:
+        print("❌ Unable to load annotation data, exiting")
+        return None, None, None, None, None
+
+    # Build document index with sentence-based chunking
+    print("\n🔄 Building document index with sentence-based chunking...")
+    document_index = build_document_index(annotations, chunk_size=256, chunk_overlap=25)
+
+    # Create tag embeddings
+    print("\n🔄 Creating tag embeddings...")
+    tag_embeddings = create_tag_embeddings(embedding_model, document_index)
+
+    # Create document-tag mapping
+    print("\n🔄 Creating document-tag mapping...")
+    doc_tag_mapping = create_document_tag_mapping(document_index, tag_embeddings)
+
+    # Create chunk embeddings if enabled
+    chunk_embeddings = None
+    if enable_chunk_embeddings:
+        print("\n🔄 Creating chunk embeddings...")
+        chunk_embeddings = create_chunk_embeddings(embedding_model, document_index)
+
+    # Save the system
+    print("\n💾 Saving document system...")
+    save_document_system(document_index, tag_embeddings, doc_tag_mapping, chunk_embeddings)
+
+    print("\n✅ Medical RAG system built successfully!")
+    return embedding_model, document_index, tag_embeddings, doc_tag_mapping, chunk_embeddings
+
+
+def demo_rag_query(query: str = "chest pain and shortness of breath", 
+                  strategy: str = "top_p", use_chunks: bool = True, **kwargs):
+    """Demo RAG query functionality with different selection strategies."""
+    print(f"\n🔍 Demo Query: '{query}' (Strategy: {strategy}, Use chunks: {use_chunks})")
+    print("=" * 60)
+    
+    # Try to load existing system first
+    load_result = load_document_system()
+    if len(load_result) == 4:
+        document_index, tag_embeddings, doc_tag_mapping, chunk_embeddings = load_result
+    else:
+        document_index, tag_embeddings, doc_tag_mapping = load_result[:3]
+        chunk_embeddings = None
+    
+    if document_index is None:
+        print("📦 No saved system found, building new one...")
+        build_result = build_medical_rag_system(enable_chunk_embeddings=use_chunks)
+        if build_result[0] is None:
+            return
+        embedding_model, document_index, tag_embeddings, doc_tag_mapping, chunk_embeddings = build_result
+    else:
+        embedding_model = load_biomedbert_model()
+    
+    # Find relevant documents using specified strategy
+    relevant_docs = find_relevant_documents(
+        query, embedding_model, tag_embeddings, doc_tag_mapping, 
+        strategy=strategy, **kwargs
+    )
+    
+    if use_chunks and chunk_embeddings:
+        # Find relevant chunks within the selected documents
+        print(f"\n🔍 Finding relevant chunks within selected documents...")
+        relevant_chunks = find_relevant_chunks(
+            query, embedding_model, relevant_docs, chunk_embeddings, top_chunks_per_doc=3
+        )
+        
+        # Get chunks for RAG
+        rag_content = get_chunks_for_rag(relevant_chunks, max_chunks=10)
+        print(f"\n📋 Ready for RAG with {len(rag_content)} chunks")
+        
+    else:
+        # Get full documents for RAG
+        rag_content = get_documents_for_rag(relevant_docs, document_index)
+        print(f"\n📋 Ready for RAG with {len(rag_content)} full documents")
+    
+    print("Next step: Feed this content to your LLM for answer generation")
+    return rag_content
+
+
+def demo_all_strategies(query: str = "chest pain and shortness of breath"):
+    """Demo all selection strategies for comparison."""
+    print(f"\n🔬 Comparing All Selection Strategies")
+    print("=" * 80)
+    
+    # Load system
+    document_index, tag_embeddings, doc_tag_mapping, chunk_embeddings = load_document_system()
+    if document_index is None:
+        print("📦 Building system first...")
+        embedding_model, document_index, tag_embeddings, doc_tag_mapping, chunk_embeddings = build_medical_rag_system()
+        if document_index is None:
+            return
+    else:
+        embedding_model = load_biomedbert_model()
+    
+    strategies = [
+        ("top_k", {"top_k": 3}),
+        ("top_p", {"top_p": 0.8, "min_similarity": 0.3}),
+        ("threshold", {"similarity_threshold": 0.5})
+    ]
+    
+    results = {}
+    for strategy, params in strategies:
+        print(f"\n{'='*20} {strategy.upper()} Strategy {'='*20}")
+        relevant_docs = find_relevant_documents(
+            query, embedding_model, tag_embeddings, doc_tag_mapping,
+            strategy=strategy, **params
+        )
+        results[strategy] = relevant_docs
+    
+    # Summary comparison
+    print(f"\n📊 Strategy Comparison Summary:")
+    print("-" * 50)
+    for strategy, docs in results.items():
+        print(f"{strategy:>10}: {len(docs)} documents selected")
+    
+    return results

src/pdf-version/generate_embeddings.py

@@ -0,0 +1,45 @@
+#!/usr/bin/env python3
+"""
+Quick script to generate new embeddings with sentence-based chunking
+"""
+
+import sys
+from pathlib import Path
+
+# Add src directory to Python path
+sys.path.insert(0, str(Path(__file__).parent / "src"))
+
+from src.demos.demo_runner import build_medical_rag_system
+
+def main():
+    print("🚀 Starting to build medical RAG system with new sentence-based chunking...")
+    print("📋 Configuration:")
+    print("   - Chunk size: 256 tokens")
+    print("   - Chunk overlap: 25 tokens (10%)")
+    print("   - Method: SentenceSplitter")
+    print("   - Enhanced tag embeddings: ✅")
+    print("   - Chunk embeddings: ✅")
+    print("")
+    
+    try:
+        result = build_medical_rag_system(enable_chunk_embeddings=True)
+        
+        if result[0] is not None:
+            print("✅ Successfully built medical RAG system!")
+            print("📁 Generated files:")
+            print("   - document_index.json")
+            print("   - tag_embeddings.json") 
+            print("   - document_tag_mapping.json")
+            print("   - chunk_embeddings.json")
+        else:
+            print("❌ Failed to build system")
+            
+    except KeyboardInterrupt:
+        print("\n⚠️ Process interrupted by user")
+    except Exception as e:
+        print(f"❌ Error occurred: {e}")
+        import traceback
+        traceback.print_exc()
+
+if __name__ == "__main__":
+    main()

src/pdf-version/indexing/__init__.py

@@ -0,0 +1,11 @@
+"""Document indexing and embedding generation."""
+
+from .document_indexer import build_document_index, split_text_into_chunks
+from .embedding_creator import create_text_embedding, create_tag_embeddings, create_chunk_embeddings
+from .storage import save_document_system, load_document_system
+
+__all__ = [
+    'build_document_index', 'split_text_into_chunks',
+    'create_text_embedding', 'create_tag_embeddings', 'create_chunk_embeddings',
+    'save_document_system', 'load_document_system'
+]

src/pdf-version/indexing/document_indexer.py

@@ -0,0 +1,100 @@
+"""Document indexing and chunking functionality."""
+
+import os
+from typing import List, Dict
+from llama_index.core import Document
+from llama_index.core.node_parser import SentenceSplitter
+from src.data.pdf_processing import extract_pdf_content_enhanced
+
+
+def split_text_into_chunks(text: str, chunk_size: int = 256, chunk_overlap: int = 25) -> List[Dict]:
+    """Split text into sentence-based chunks with token control.
+    
+    Args:
+        text: Input text to split.
+        chunk_size: Maximum size of each chunk in tokens.
+        chunk_overlap: Number of overlapping tokens between chunks.
+        
+    Returns:
+        List of chunk dictionaries with metadata.
+    """
+    if not text.strip():
+        return []
+    
+    # Use LlamaIndex SentenceSplitter for sentence-aware, token-based chunking
+    splitter = SentenceSplitter(
+        chunk_size=chunk_size,
+        chunk_overlap=chunk_overlap,
+        paragraph_separator="\n\n",
+        secondary_chunking_regex="[^.!?]+[.!?]"  # Split on sentences
+    )
+    
+    # Create a Document object for the splitter
+    document = Document(text=text)
+    
+    # Split the document into nodes
+    nodes = splitter.get_nodes_from_documents([document])
+    
+    # Convert nodes to our chunk format
+    chunks = []
+    for i, node in enumerate(nodes):
+        chunk_text = node.get_content()
+        if chunk_text.strip():
+            chunks.append({
+                'text': chunk_text,
+                'chunk_id': i,
+                'token_count': len(chunk_text.split()),  # Approximate token count
+                'node_id': node.node_id,
+                'start_char': getattr(node, 'start_char_idx', 0),
+                'end_char': getattr(node, 'end_char_idx', len(chunk_text))
+            })
+    
+    return chunks
+
+
+def build_document_index(annotations: List[Dict], assets_dir: str = "assets", 
+                        chunk_size: int = 256, chunk_overlap: int = 25) -> Dict:
+    """Build a comprehensive document index with sentence-based chunked content and tags.
+    
+    Args:
+        annotations: List of annotation dictionaries.
+        assets_dir: Directory containing PDF files.
+        chunk_size: Maximum size of each chunk in tokens.
+        chunk_overlap: Number of overlapping tokens between chunks.
+        
+    Returns:
+        Dictionary containing document index with chunks and metadata.
+    """
+    document_index = {}
+    
+    for item in annotations:
+        pdf_name = item['pdf']
+        pdf_path = os.path.join(assets_dir, pdf_name)
+        
+        if not os.path.exists(pdf_path):
+            print(f"⚠️ Skipping missing file: {pdf_name}")
+            continue
+            
+        print(f"🔄 Indexing document: {pdf_name}")
+        
+        # Extract full document content
+        documents = extract_pdf_content_enhanced(pdf_path)
+        full_text = "\n\n".join([doc.text for doc in documents])
+        
+        # Split into chunks
+        chunks = split_text_into_chunks(full_text, chunk_size, chunk_overlap)
+        
+        # Build comprehensive document record
+        document_index[pdf_name] = {
+            'full_content': full_text,
+            'chunks': chunks,
+            'symptoms': item.get('symptoms', []),
+            'diagnoses': item.get('diagnoses', []),
+            'treatments': item.get('treatments', []),
+            'all_tags': item.get('symptoms', []) + item.get('diagnoses', []) + item.get('treatments', [])
+        }
+        
+        print(f"  📄 Split into {len(chunks)} chunks")
+    
+    print(f"✅ Built index for {len(document_index)} documents")
+    return document_index

src/pdf-version/indexing/embedding_creator.py

@@ -0,0 +1,108 @@
+"""Embedding generation for tags and document chunks."""
+
+from typing import Dict
+import numpy as np
+from sentence_transformers import SentenceTransformer
+
+
+def create_text_embedding(model: SentenceTransformer, text: str) -> np.ndarray:
+    """Create embedding for a single text.
+    
+    Args:
+        model: SentenceTransformer model.
+        text: Input text.
+        
+    Returns:
+        Numpy array containing the embedding.
+    """
+    if not text.strip():
+        return np.zeros(model.get_sentence_embedding_dimension())
+    return model.encode([text])[0]
+
+
+def create_tag_embeddings(model: SentenceTransformer, document_index: Dict) -> Dict:
+    """Create enhanced embeddings for all unique tags with medical context.
+    
+    Args:
+        model: SentenceTransformer model.
+        document_index: Document index dictionary.
+        
+    Returns:
+        Dictionary mapping tags to their embeddings.
+    """
+    all_tags = set()
+    
+    # Collect all unique tags
+    for doc_info in document_index.values():
+        all_tags.update(doc_info['all_tags'])
+    
+    print(f"🔄 Creating enhanced embeddings for {len(all_tags)} unique tags")
+    
+    tag_embeddings = {}
+    for tag in all_tags:
+        if tag.strip():
+            # Original tag embedding
+            base_embedding = create_text_embedding(model, tag)
+            
+            # Medical context variations
+            contexts = [
+                f"patient presents with {tag}",
+                f"clinical manifestation of {tag}",
+                f"emergency department patient has {tag}",
+                f"medical condition: {tag}"
+            ]
+            
+            # Generate context embeddings
+            context_embeddings = []
+            for ctx in contexts:
+                ctx_emb = create_text_embedding(model, ctx)
+                context_embeddings.append(ctx_emb)
+            
+            # Combine original + context embeddings (weighted average)
+            all_embeddings = [base_embedding] + context_embeddings
+            enhanced_embedding = np.mean(all_embeddings, axis=0)
+            
+            tag_embeddings[tag] = enhanced_embedding
+    
+    print(f"✅ Created {len(tag_embeddings)} enhanced tag embeddings with medical context")
+    return tag_embeddings
+
+
+def create_chunk_embeddings(model: SentenceTransformer, document_index: Dict) -> Dict:
+    """Create embeddings for all document chunks.
+    
+    Args:
+        model: SentenceTransformer model.
+        document_index: Document index dictionary.
+        
+    Returns:
+        Dictionary mapping document names to their chunk embeddings.
+    """
+    chunk_embeddings = {}
+    total_chunks = 0
+    
+    print("🔄 Creating chunk embeddings...")
+    
+    for pdf_name, doc_info in document_index.items():
+        chunks = doc_info['chunks']
+        doc_chunk_embeddings = []
+        
+        for chunk in chunks:
+            chunk_text = chunk['text']
+            if chunk_text.strip():
+                embedding = create_text_embedding(model, chunk_text)
+                doc_chunk_embeddings.append({
+                    'chunk_id': chunk['chunk_id'],
+                    'text': chunk_text,
+                    'start_char': chunk.get('start_char', 0),
+                    'end_char': chunk.get('end_char', len(chunk_text)),
+                    'token_count': chunk.get('token_count', len(chunk_text.split())),
+                    'embedding': embedding
+                })
+        
+        chunk_embeddings[pdf_name] = doc_chunk_embeddings
+        total_chunks += len(doc_chunk_embeddings)
+        print(f"  📄 {pdf_name}: {len(doc_chunk_embeddings)} chunks")
+    
+    print(f"✅ Created embeddings for {total_chunks} chunks across all documents")
+    return chunk_embeddings

src/pdf-version/indexing/storage.py

@@ -0,0 +1,149 @@
+"""Data persistence for document system."""
+
+import json
+import os
+from typing import Dict, Optional, Tuple
+import numpy as np
+
+
+def save_document_system(document_index: Dict, tag_embeddings: Dict, 
+                        doc_tag_mapping: Dict, chunk_embeddings: Dict = None, 
+                        output_dir: str = "."):
+    """Save the complete document indexing system.
+    
+    Args:
+        document_index: Document index dictionary.
+        tag_embeddings: Tag embeddings dictionary.
+        doc_tag_mapping: Document-tag mapping dictionary.
+        chunk_embeddings: Chunk embeddings dictionary (optional).
+        output_dir: Output directory for saved files.
+    """
+    
+    # Save document index (content + metadata + chunks)
+    doc_index_serializable = {}
+    for doc_name, doc_info in document_index.items():
+        doc_index_serializable[doc_name] = {
+            'full_content': doc_info.get('full_content', doc_info.get('content', '')),
+            'chunks': doc_info.get('chunks', []),
+            'symptoms': doc_info['symptoms'],
+            'diagnoses': doc_info['diagnoses'],
+            'treatments': doc_info.get('treatments', []),
+            'all_tags': doc_info['all_tags']
+        }
+    
+    with open(os.path.join(output_dir, 'document_index.json'), 'w', encoding='utf-8') as f:
+        json.dump(doc_index_serializable, f, indent=2, ensure_ascii=False)
+    
+    # Save tag embeddings
+    tag_embeddings_serializable = {
+        tag: embedding.tolist() for tag, embedding in tag_embeddings.items()
+    }
+    with open(os.path.join(output_dir, 'tag_embeddings.json'), 'w', encoding='utf-8') as f:
+        json.dump(tag_embeddings_serializable, f, indent=2, ensure_ascii=False)
+    
+    # Save document-tag mapping
+    doc_tag_serializable = {}
+    for doc_name, doc_info in doc_tag_mapping.items():
+        doc_tag_serializable[doc_name] = {
+            'tags': doc_info['tags'],
+            'symptoms': doc_info['symptoms'],
+            'diagnoses': doc_info['diagnoses'],
+            'treatments': doc_info['treatments'],
+            'tag_embeddings': {
+                tag: embedding.tolist() 
+                for tag, embedding in doc_info['tag_embeddings'].items()
+            }
+        }
+    
+    with open(os.path.join(output_dir, 'document_tag_mapping.json'), 'w', encoding='utf-8') as f:
+        json.dump(doc_tag_serializable, f, indent=2, ensure_ascii=False)
+    
+    # Save chunk embeddings if provided
+    if chunk_embeddings:
+        chunk_embeddings_serializable = {}
+        for doc_name, chunks in chunk_embeddings.items():
+            chunk_embeddings_serializable[doc_name] = []
+            for chunk in chunks:
+                chunk_embeddings_serializable[doc_name].append({
+                    'chunk_id': chunk['chunk_id'],
+                    'text': chunk['text'],
+                    'start_char': chunk.get('start_char', 0),
+                    'end_char': chunk.get('end_char', len(chunk['text'])),
+                    'token_count': chunk.get('token_count', len(chunk['text'].split())),
+                    'embedding': chunk['embedding'].tolist()
+                })
+        
+        with open(os.path.join(output_dir, 'chunk_embeddings.json'), 'w', encoding='utf-8') as f:
+            json.dump(chunk_embeddings_serializable, f, indent=2, ensure_ascii=False)
+    
+    print("✅ Document system saved to files")
+
+
+def load_document_system(input_dir: str = ".") -> Tuple[Optional[Dict], Optional[Dict], Optional[Dict], Optional[Dict]]:
+    """Load the complete document indexing system.
+    
+    Args:
+        input_dir: Input directory containing saved files.
+        
+    Returns:
+        Tuple of (document_index, tag_embeddings, doc_tag_mapping, chunk_embeddings).
+        Returns (None, None, None, None) if loading fails.
+    """
+    try:
+        # Load document index
+        with open(os.path.join(input_dir, 'document_index.json'), 'r', encoding='utf-8') as f:
+            document_index = json.load(f)
+        
+        # Load tag embeddings
+        with open(os.path.join(input_dir, 'tag_embeddings.json'), 'r', encoding='utf-8') as f:
+            tag_embeddings_data = json.load(f)
+            tag_embeddings = {
+                tag: np.array(embedding) 
+                for tag, embedding in tag_embeddings_data.items()
+            }
+        
+        # Load document-tag mapping
+        with open(os.path.join(input_dir, 'document_tag_mapping.json'), 'r', encoding='utf-8') as f:
+            doc_tag_data = json.load(f)
+            doc_tag_mapping = {}
+            for doc_name, doc_info in doc_tag_data.items():
+                doc_tag_mapping[doc_name] = {
+                    'tags': doc_info['tags'],
+                    'symptoms': doc_info['symptoms'],
+                    'diagnoses': doc_info['diagnoses'],
+                    'treatments': doc_info['treatments'],
+                    'tag_embeddings': {
+                        tag: np.array(embedding)
+                        for tag, embedding in doc_info['tag_embeddings'].items()
+                    }
+                }
+        
+        # Try to load chunk embeddings if they exist
+        chunk_embeddings = None
+        chunk_embeddings_path = os.path.join(input_dir, 'chunk_embeddings.json')
+        if os.path.exists(chunk_embeddings_path):
+            with open(chunk_embeddings_path, 'r', encoding='utf-8') as f:
+                chunk_data = json.load(f)
+                chunk_embeddings = {}
+                for doc_name, chunks in chunk_data.items():
+                    chunk_embeddings[doc_name] = []
+                    for chunk in chunks:
+                        chunk_embeddings[doc_name].append({
+                            'chunk_id': chunk['chunk_id'],
+                            'text': chunk['text'],
+                            'start_char': chunk.get('start_char', 0),
+                            'end_char': chunk.get('end_char', len(chunk['text'])),
+                            'token_count': chunk.get('token_count', len(chunk['text'].split())),
+                            # Backward compatibility for old format
+                            'start_word': chunk.get('start_word', 0),
+                            'end_word': chunk.get('end_word', len(chunk['text'].split())),
+                            'embedding': np.array(chunk['embedding'])
+                        })
+            print("✅ Chunk embeddings loaded")
+        
+        print("✅ Document system loaded successfully")
+        return document_index, tag_embeddings, doc_tag_mapping, chunk_embeddings
+        
+    except Exception as e:
+        print(f"❌ Failed to load document system: {e}")
+        return None, None, None, None

src/pdf-version/main.py

@@ -0,0 +1,83 @@
+#!/usr/bin/env python3
+"""OnCall AI - Medical RAG System
+
+Main entry point for the medical RAG system.
+"""
+
+import sys
+from pathlib import Path
+
+# Add src directory to Python path
+sys.path.insert(0, str(Path(__file__).parent / "src"))
+
+from src.demos.demo_runner import build_medical_rag_system, demo_rag_query, demo_all_strategies
+
+
+def main():
+    """Main program entry point."""
+    try:
+        # Build the system with chunk embeddings
+        build_medical_rag_system(enable_chunk_embeddings=True)
+        
+        # Demo chunk-based retrieval
+        print("\n" + "="*80)
+        print("🧩 CHUNK-BASED RETRIEVAL DEMO")
+        print("="*80)
+        demo_rag_query("chest pain and shortness of breath", 
+                      strategy="top_p", use_chunks=True, top_p=0.8)
+        
+    except KeyboardInterrupt:
+        print("\n\n👋 User interrupted, program exiting")
+    except Exception as e:
+        print(f"\n❌ Program execution error: {e}")
+        import traceback
+        traceback.print_exc()
+
+
+def interactive_demo():
+    """Interactive demo mode."""
+    print("🏥 OnCall AI - Interactive Demo Mode")
+    print("=" * 50)
+    
+    while True:
+        print("\nOptions:")
+        print("1. Build/rebuild system")
+        print("2. Query with TOP-P strategy")
+        print("3. Query with TOP-K strategy") 
+        print("4. Compare all strategies")
+        print("5. Custom query")
+        print("6. Exit")
+        
+        choice = input("\nSelect option (1-6): ").strip()
+        
+        if choice == "1":
+            build_medical_rag_system(enable_chunk_embeddings=True)
+        elif choice == "2":
+            query = input("Enter your query: ").strip()
+            if query:
+                demo_rag_query(query, strategy="top_p", use_chunks=True)
+        elif choice == "3":
+            query = input("Enter your query: ").strip()
+            if query:
+                demo_rag_query(query, strategy="top_k", use_chunks=True, top_k=3)
+        elif choice == "4":
+            query = input("Enter your query: ").strip()
+            if query:
+                demo_all_strategies(query)
+        elif choice == "5":
+            query = input("Enter your query: ").strip()
+            strategy = input("Enter strategy (top_k/top_p/threshold): ").strip()
+            if query and strategy:
+                demo_rag_query(query, strategy=strategy, use_chunks=True)
+        elif choice == "6":
+            print("👋 Goodbye!")
+            break
+        else:
+            print("❌ Invalid option. Please select 1-6.")
+
+
+if __name__ == "__main__":
+    if len(sys.argv) > 1 and sys.argv[1] == "--interactive":
+        interactive_demo()
+    else:
+        main()

src/pdf-version/models/__init__.py

@@ -0,0 +1,5 @@
+"""Model loading and management."""
+
+from .embedding_models import load_biomedbert_model, load_meditron_model
+
+__all__ = ['load_biomedbert_model', 'load_meditron_model']

src/pdf-version/models/embedding_models.py

@@ -0,0 +1,56 @@
+"""Embedding model loading and management."""
+
+from typing import Optional
+import torch
+from sentence_transformers import SentenceTransformer, models
+
+
+def load_biomedbert_model(device: Optional[str] = None) -> SentenceTransformer:
+    """Load BGE Large Medical model optimized for medical domain embeddings.
+    
+    Args:
+        device: Device to use ('cuda', 'mps', 'cpu'). Auto-detects if None.
+        
+    Returns:
+        Loaded SentenceTransformer model.
+    """
+    if device is None:
+        if torch.cuda.is_available():
+            device = "cuda"
+        elif torch.backends.mps.is_available():  # Apple Silicon GPU
+            device = "mps"
+        else:
+            device = "cpu"
+
+    print(f"Using device: {device}")
+    
+    # Use BGE Large Medical which is optimized for medical domain
+    try:
+        model = SentenceTransformer('ls-da3m0ns/bge_large_medical')
+        model = model.to(device)
+        print("✅ Loaded BGE Large Medical model for medical embeddings")
+        return model
+    except Exception as e:
+        print(f"❌ Failed to load BGE Large Medical: {e}")
+        print("🔄 Falling back to manual construction...")
+        
+        # Fallback to manual construction if direct loading fails
+        word_embedding_model = models.Transformer('ls-da3m0ns/bge_large_medical')
+        pooling_model = models.Pooling(word_embedding_model.get_word_embedding_dimension())
+        model = SentenceTransformer(modules=[word_embedding_model, pooling_model])
+        model = model.to(device)
+        return model
+
+
+def load_meditron_model():
+    """Load Meditron-7B model (placeholder for future implementation).
+    
+    Returns:
+        None (not implemented yet).
+    """
+    # TODO: Implement Meditron-7B loading
+    # from transformers import AutoTokenizer, AutoModelForCausalLM
+    # tokenizer = AutoTokenizer.from_pretrained("epfl-llm/meditron-7b")
+    # model = AutoModelForCausalLM.from_pretrained("epfl-llm/meditron-7b")
+    print("Meditron-7B to be implemented")
+    return None

src/pdf-version/oncall_ai.py

@@ -0,0 +1,55 @@
+#!/usr/bin/env python3
+"""OnCall AI - Medical RAG System (Backward Compatibility)
+
+This file provides backward compatibility with the original rag.py interface.
+Import everything from the new modular structure.
+"""
+
+import sys
+from pathlib import Path
+
+# Add src directory to Python path
+sys.path.insert(0, str(Path(__file__).parent / "src"))
+
+# Import all functions for backward compatibility
+from src.models.embedding_models import load_biomedbert_model, load_meditron_model
+from src.data.loaders import load_annotations, filter_pdf_files
+from src.data.pdf_processing import (
+    extract_pdf_text, extract_tables_from_pdf,
+    extract_images_ocr_from_pdf, extract_pdf_content_enhanced
+)
+from src.indexing.document_indexer import build_document_index, split_text_into_chunks
+from src.indexing.embedding_creator import create_text_embedding, create_tag_embeddings, create_chunk_embeddings
+from src.indexing.storage import save_document_system, load_document_system
+from src.retrieval.document_retriever import (
+    find_relevant_documents_top_k, find_relevant_documents_top_p,
+    find_relevant_documents_threshold, find_relevant_documents,
+    create_document_tag_mapping
+)
+from src.retrieval.chunk_retriever import find_relevant_chunks, get_documents_for_rag, get_chunks_for_rag
+from src.demos.demo_runner import build_medical_rag_system, demo_rag_query, demo_all_strategies
+
+# Main function for backward compatibility
+def main():
+    """Main program entry compatible with original rag.py."""
+    try:
+        # Build the system with chunk embeddings
+        build_medical_rag_system(enable_chunk_embeddings=True)
+        
+        # Demo chunk-based retrieval
+        print("\n" + "="*80)
+        print("🧩 CHUNK-BASED RETRIEVAL DEMO")
+        print("="*80)
+        demo_rag_query("chest pain and shortness of breath", 
+                      strategy="top_p", use_chunks=True, top_p=0.8)
+        
+    except KeyboardInterrupt:
+        print("\n\n👋 User interrupted, program exiting")
+    except Exception as e:
+        print(f"\n❌ Program execution error: {e}")
+        import traceback
+        traceback.print_exc()
+
+
+if __name__ == "__main__":
+    main()

src/pdf-version/rag/__init__.py

@@ -0,0 +1,23 @@
+"""Medical RAG Pipeline module (Functional Programming)."""
+
+from .medical_rag_pipeline import (
+    generate_with_ollama,
+    retrieve_medical_context,
+    evaluate_context_quality,
+    create_medical_prompt,
+    generate_medical_response,
+    answer_medical_query,
+    load_rag_data,
+    quick_medical_query
+)
+
+__all__ = [
+    'generate_with_ollama',
+    'retrieve_medical_context', 
+    'evaluate_context_quality',
+    'create_medical_prompt',
+    'generate_medical_response',
+    'answer_medical_query',
+    'load_rag_data',
+    'quick_medical_query'
+]

src/pdf-version/rag/medical_rag_pipeline.py

@@ -0,0 +1,457 @@
+"""Complete Medical RAG Pipeline integrating retrieval system with Meditron-7B (Functional Programming)."""
+
+import json
+import requests
+import numpy as np
+from typing import Dict, List, Optional, Tuple
+from sentence_transformers import SentenceTransformer
+
+# Import existing retrieval components
+import sys
+import os
+sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
+
+from retrieval.document_retriever import find_relevant_documents
+from retrieval.chunk_retriever import find_relevant_chunks, get_chunks_for_rag
+from models.embedding_models import load_biomedbert_model
+
+
+def generate_with_ollama(prompt: str, 
+                        model: str = "meditron:7b",
+                        base_url: str = "http://localhost:11434",
+                        temperature: float = 0.1, 
+                        max_tokens: int = 300) -> Dict:
+    """Generate response using Ollama model.
+    
+    Args:
+        prompt: Input prompt for the model
+        model: Ollama model name
+        base_url: Ollama server URL
+        temperature: Sampling temperature
+        max_tokens: Maximum tokens to generate
+        
+    Returns:
+        Dictionary with response or error
+    """
+    url = f"{base_url}/api/generate"
+    
+    payload = {
+        "model": model,
+        "prompt": prompt,
+        "stream": False,
+        "options": {
+            "temperature": temperature,
+            "num_predict": max_tokens,
+            "top_p": 0.9,
+            "top_k": 40
+        }
+    }
+    
+    try:
+        response = requests.post(url, json=payload, timeout=120)
+        response.raise_for_status()
+        return response.json()
+    except requests.exceptions.RequestException as e:
+        return {"error": f"LLM request failed: {str(e)}"}
+
+
+def retrieve_medical_context(query: str,
+                           embedding_model: SentenceTransformer,
+                           tag_embeddings: Dict,
+                           chunk_embeddings: Dict,
+                           doc_tag_mapping: Dict,
+                           doc_strategy: str = "top_p",
+                           chunk_strategy: str = "top_p",
+                           max_chunks: int = 5) -> Dict:
+    """
+    Retrieve relevant medical context for query using two-stage retrieval.
+    
+    Args:
+        query: Medical question/query
+        embedding_model: BGE Large Medical model
+        tag_embeddings: Pre-computed tag embeddings
+        chunk_embeddings: Pre-computed chunk embeddings
+        doc_tag_mapping: Document to tag mapping
+        doc_strategy: Document retrieval strategy
+        chunk_strategy: Chunk retrieval strategy
+        max_chunks: Maximum chunks to retrieve
+        
+    Returns:
+        Dictionary with retrieval results and metadata
+    """
+    print(f"🔍 Retrieving context for: '{query}'")
+    
+    # Stage 1: Document-level retrieval
+    print("📄 Stage 1: Document retrieval...")
+    relevant_docs = find_relevant_documents(
+        query, embedding_model, tag_embeddings, doc_tag_mapping,
+        strategy=doc_strategy, top_p=0.6, min_similarity=0.5
+    )
+    
+    if not relevant_docs:
+        print("⚠️ No relevant documents found")
+        return {
+            "has_context": False,
+            "relevant_documents": [],
+            "relevant_chunks": [],
+            "rag_chunks": [],
+            "context_text": "",
+            "retrieval_metadata": {
+                "total_docs": 0,
+                "total_chunks": 0,
+                "context_length": 0
+            }
+        }
+    
+    # Stage 2: Chunk-level retrieval
+    print("📝 Stage 2: Chunk retrieval...")
+    relevant_chunks = find_relevant_chunks(
+        query, embedding_model, relevant_docs, chunk_embeddings,
+        strategy=chunk_strategy, top_p=0.6, min_similarity=0.3, 
+        similarity_metric="dot_product"
+    )
+    
+    if not relevant_chunks:
+        print("⚠️ No relevant chunks found")
+        return {
+            "has_context": False,
+            "relevant_documents": relevant_docs,
+            "relevant_chunks": [],
+            "rag_chunks": [],
+            "context_text": "",
+            "retrieval_metadata": {
+                "total_docs": len(relevant_docs),
+                "total_chunks": 0,
+                "context_length": 0
+            }
+        }
+    
+    # Stage 3: Prepare RAG context
+    print("🎯 Stage 3: Preparing RAG context...")
+    rag_chunks = get_chunks_for_rag(relevant_chunks, max_chunks)
+    context_text = "\n\n".join(rag_chunks)
+    
+    # Calculate retrieval statistics
+    avg_similarity = np.mean([chunk['similarity'] for chunk in relevant_chunks])
+    max_similarity = max([chunk['similarity'] for chunk in relevant_chunks])
+    
+    print(f"✅ Context prepared: {len(rag_chunks)} chunks, avg_sim={avg_similarity:.3f}")
+    
+    return {
+        "has_context": True,
+        "relevant_documents": relevant_docs,
+        "relevant_chunks": relevant_chunks,
+        "rag_chunks": rag_chunks,
+        "context_text": context_text,
+        "retrieval_metadata": {
+            "total_docs": len(relevant_docs),
+            "total_chunks": len(relevant_chunks),
+            "chunks_for_rag": len(rag_chunks),
+            "context_length": len(context_text),
+            "avg_similarity": float(avg_similarity),
+            "max_similarity": float(max_similarity)
+        }
+    }
+
+
+def evaluate_context_quality(context_result: Dict, query: str) -> Dict:
+    """
+    Evaluate if retrieved context is sufficient to answer the query.
+    
+    Args:
+        context_result: Result from retrieve_medical_context()
+        query: Original query
+        
+    Returns:
+        Quality assessment dictionary
+    """
+    if not context_result["has_context"]:
+        return {
+            "is_sufficient": False,
+            "confidence": 0.0,
+            "reason": "No relevant medical documents found in database"
+        }
+    
+    metadata = context_result["retrieval_metadata"]
+    
+    # Quality heuristics
+    min_similarity_threshold = 0.4
+    min_chunks_threshold = 2
+    min_context_length = 200
+    
+    # Check similarity scores
+    avg_sim = metadata["avg_similarity"]
+    max_sim = metadata["max_similarity"]
+    
+    # Check quantity
+    chunk_count = metadata["chunks_for_rag"]
+    context_length = metadata["context_length"]
+    
+    # Determine if context is sufficient
+    quality_checks = {
+        "high_similarity": max_sim >= min_similarity_threshold,
+        "sufficient_chunks": chunk_count >= min_chunks_threshold,
+        "sufficient_length": context_length >= min_context_length,
+        "decent_avg_similarity": avg_sim >= 0.3
+    }
+    
+    passed_checks = sum(quality_checks.values())
+    confidence = passed_checks / len(quality_checks)
+    
+    is_sufficient = passed_checks >= 3  # At least 3/4 checks must pass
+    
+    if not is_sufficient:
+        if not quality_checks["high_similarity"]:
+            reason = f"Low similarity to medical documents (max: {max_sim:.3f})"
+        elif not quality_checks["sufficient_chunks"]:
+            reason = f"Insufficient relevant content found ({chunk_count} chunks)"
+        else:
+            reason = "Retrieved content may not adequately address the query"
+    else:
+        reason = "Context appears sufficient for medical response"
+    
+    return {
+        "is_sufficient": is_sufficient,
+        "confidence": confidence,
+        "reason": reason,
+        "quality_checks": quality_checks,
+        "similarity_stats": {
+            "avg_similarity": avg_sim,
+            "max_similarity": max_sim
+        }
+    }
+
+
+def create_medical_prompt(query: str, context: str, context_quality: Dict) -> str:
+    """
+    Create a medical prompt with proper instructions and context.
+    
+    Args:
+        query: User's medical question
+        context: Retrieved medical context
+        context_quality: Context quality assessment
+        
+    Returns:
+        Formatted prompt for medical LLM
+    """
+    # Base medical prompt with professional identity
+    base_prompt = """You are a medical AI assistant. Your role is to provide accurate medical information based strictly on the provided medical literature context.
+
+IMPORTANT GUIDELINES:
+1. Base your answers ONLY on the provided medical context
+2. If the context doesn't contain sufficient information to answer the question, clearly state: "Based on the available medical literature in my database, I cannot provide a complete answer to this question."
+3. Always cite that your response is "based on the provided medical literature"
+4. Do not make assumptions or provide information not present in the context
+5. For serious medical conditions, always recommend consulting healthcare professionals
+6. Be precise and use appropriate medical terminology
+
+"""
+    
+    if context_quality["is_sufficient"]:
+        # High-confidence response with context
+        prompt = f"""{base_prompt}
+
+MEDICAL LITERATURE CONTEXT:
+{context}
+
+QUESTION: {query}
+
+MEDICAL RESPONSE (based on the provided medical literature):"""
+    
+    else:
+        # Low-confidence response with limited context
+        prompt = f"""{base_prompt}
+
+LIMITED MEDICAL CONTEXT AVAILABLE:
+{context if context else "No directly relevant medical literature found."}
+
+QUESTION: {query}
+
+MEDICAL RESPONSE: Based on the available medical literature in my database, I have limited information to fully address this question. {context_quality["reason"]}
+
+However, here is what I can provide based on the available context:"""
+    
+    return prompt
+
+
+def generate_medical_response(prompt: str, model: str = "meditron:7b") -> Dict:
+    """
+    Generate medical response using Meditron-7B.
+    
+    Args:
+        prompt: Formatted medical prompt
+        model: Ollama model name
+        
+    Returns:
+        LLM response dictionary
+    """
+    print("🧠 Generating medical response...")
+    
+    # Use low temperature for medical accuracy
+    result = generate_with_ollama(
+        prompt, 
+        model=model,
+        temperature=0.1,  # Very low for medical precision
+        max_tokens=400
+    )
+    
+    if "error" in result:
+        return {
+            "success": False,
+            "response": "I apologize, but I'm currently unable to process medical queries due to a technical issue. Please consult a healthcare professional for medical advice.",
+            "error": result["error"]
+        }
+    
+    response_text = result.get("response", "").strip()
+    
+    # Basic response validation
+    if len(response_text) < 20:
+        return {
+            "success": False,
+            "response": "I was unable to generate a meaningful response. Please rephrase your medical question or consult a healthcare professional.",
+            "error": "Generated response too short"
+        }
+    
+    return {
+        "success": True,
+        "response": response_text,
+        "generation_metadata": {
+            "model": model,
+            "response_length": len(response_text)
+        }
+    }
+
+
+def answer_medical_query(query: str,
+                        embedding_model: SentenceTransformer,
+                        tag_embeddings: Dict,
+                        chunk_embeddings: Dict,
+                        doc_tag_mapping: Dict,
+                        document_index: Dict,
+                        model: str = "meditron:7b",
+                        **kwargs) -> Dict:
+    """
+    Complete medical RAG pipeline: retrieve context and generate answer.
+    
+    Args:
+        query: Medical question
+        embedding_model: BGE Large Medical model
+        tag_embeddings: Pre-computed tag embeddings
+        chunk_embeddings: Pre-computed chunk embeddings
+        doc_tag_mapping: Document to tag mapping
+        document_index: Complete document index
+        model: Ollama model name
+        **kwargs: Additional parameters for retrieval and generation
+        
+    Returns:
+        Complete response dictionary with metadata
+    """
+    print("\n" + "="*60)
+    print(f"🏥 Medical RAG Query: '{query}'")
+    print("="*60)
+    
+    # Step 1: Retrieve medical context
+    context_result = retrieve_medical_context(
+        query, embedding_model, tag_embeddings, chunk_embeddings, 
+        doc_tag_mapping, **kwargs
+    )
+    
+    # Step 2: Evaluate context quality
+    context_quality = evaluate_context_quality(context_result, query)
+    
+    print(f"📊 Context Quality: {context_quality['confidence']:.1%} confidence")
+    print(f"💭 Assessment: {context_quality['reason']}")
+    
+    # Step 3: Create medical prompt
+    medical_prompt = create_medical_prompt(
+        query, context_result["context_text"], context_quality
+    )
+    
+    # Step 4: Generate medical response
+    response_result = generate_medical_response(medical_prompt, model)
+    
+    # Step 5: Compile complete result
+    complete_result = {
+        "query": query,
+        "answer": response_result["response"],
+        "success": response_result["success"],
+        "context_quality": context_quality,
+        "retrieval_metadata": context_result["retrieval_metadata"],
+        "sources": {
+            "documents": context_result["relevant_documents"],
+            "chunk_count": len(context_result["rag_chunks"])
+        }
+    }
+    
+    # Add error information if present
+    if "error" in response_result:
+        complete_result["error"] = response_result["error"]
+    
+    print(f"\n✅ Response generated successfully: {len(response_result['response'])} characters")
+    return complete_result
+
+
+def load_rag_data(tag_embeddings_path: str = "tag_embeddings.json",
+                  chunk_embeddings_path: str = "chunk_embeddings.json", 
+                  doc_tag_mapping_path: str = "document_tag_mapping.json",
+                  document_index_path: str = "document_index.json") -> Tuple[SentenceTransformer, Dict, Dict, Dict, Dict]:
+    """
+    Load all RAG data needed for medical question answering.
+    
+    Args:
+        tag_embeddings_path: Path to tag embeddings
+        chunk_embeddings_path: Path to chunk embeddings
+        doc_tag_mapping_path: Path to document tag mapping
+        document_index_path: Path to document index
+        
+    Returns:
+        Tuple of (embedding_model, tag_embeddings, chunk_embeddings, doc_tag_mapping, document_index)
+    """
+    print("🔄 Loading Medical RAG Data...")
+    
+    # Load embedding model
+    print("📦 Loading BGE Large Medical embedding model...")
+    embedding_model = load_biomedbert_model()
+    
+    # Load embeddings and indices
+    print("📂 Loading embeddings and indices...")
+    
+    with open(tag_embeddings_path, 'r') as f:
+        tag_embeddings = json.load(f)
+        tag_embeddings = {tag: np.array(embedding) for tag, embedding in tag_embeddings.items()}
+    
+    with open(chunk_embeddings_path, 'r') as f:
+        chunk_embeddings = json.load(f)
+        for doc_name, chunks in chunk_embeddings.items():
+            for chunk in chunks:
+                chunk['embedding'] = np.array(chunk['embedding'])
+    
+    with open(doc_tag_mapping_path, 'r') as f:
+        doc_tag_mapping = json.load(f)
+    
+    with open(document_index_path, 'r') as f:
+        document_index = json.load(f)
+    
+    print("✅ Medical RAG data loaded successfully!")
+    return embedding_model, tag_embeddings, chunk_embeddings, doc_tag_mapping, document_index
+
+
+def quick_medical_query(query: str, max_chunks: int = 3) -> Dict:
+    """
+    Quick medical query with default settings.
+    
+    Args:
+        query: Medical question
+        max_chunks: Maximum chunks to retrieve
+        
+    Returns:
+        Medical response dictionary
+    """
+    # Load data
+    embedding_model, tag_embeddings, chunk_embeddings, doc_tag_mapping, document_index = load_rag_data()
+    
+    # Answer query
+    return answer_medical_query(
+        query, embedding_model, tag_embeddings, chunk_embeddings,
+        doc_tag_mapping, document_index, max_chunks=max_chunks
+    )

src/pdf-version/retrieval/__init__.py

@@ -0,0 +1,17 @@
+"""Retrieval systems for documents and chunks."""
+
+from .document_retriever import (
+    find_relevant_documents_top_k,
+    find_relevant_documents_top_p, 
+    find_relevant_documents_threshold,
+    find_relevant_documents,
+    create_document_tag_mapping
+)
+from .chunk_retriever import find_relevant_chunks, get_documents_for_rag, get_chunks_for_rag
+
+__all__ = [
+    'find_relevant_documents_top_k', 'find_relevant_documents_top_p',
+    'find_relevant_documents_threshold', 'find_relevant_documents',
+    'create_document_tag_mapping', 'find_relevant_chunks',
+    'get_documents_for_rag', 'get_chunks_for_rag'
+]

src/pdf-version/retrieval/chunk_retriever.py

@@ -0,0 +1,193 @@
+"""Chunk-level retrieval functionality."""
+
+from typing import List, Dict, Callable
+import numpy as np
+from sentence_transformers import SentenceTransformer
+from src.indexing.embedding_creator import create_text_embedding
+
+
+def cosine_similarity(vec1: np.ndarray, vec2: np.ndarray) -> float:
+    """Calculate cosine similarity between two vectors."""
+    return np.dot(vec1, vec2) / (np.linalg.norm(vec1) * np.linalg.norm(vec2))
+
+
+def dot_product_similarity(vec1: np.ndarray, vec2: np.ndarray) -> float:
+    """Calculate dot product similarity for normalized vectors."""
+    # For normalized vectors (like BGE embeddings), dot product = cosine similarity
+    # This is computationally more efficient than cosine similarity
+    return np.dot(vec1, vec2)
+
+
+# Similarity function registry
+SIMILARITY_FUNCTIONS = {
+    "cosine": cosine_similarity,
+    "dot_product": dot_product_similarity
+}
+
+
+def find_relevant_chunks_top_k(query: str, model: SentenceTransformer, 
+                              relevant_docs: List[str], chunk_embeddings: Dict, 
+                              top_chunks_per_doc: int = 3, 
+                              similarity_metric: str = "cosine") -> List[Dict]:
+    """Find most relevant chunks using Top-K strategy (original method)."""
+    query_embedding = create_text_embedding(model, query)
+    
+    all_relevant_chunks = []
+    
+    for doc_name in relevant_docs:
+        if doc_name not in chunk_embeddings:
+            continue
+            
+        doc_chunks = chunk_embeddings[doc_name]
+        chunk_similarities = []
+        
+        # Get similarity function
+        similarity_func = SIMILARITY_FUNCTIONS.get(similarity_metric, cosine_similarity)
+        
+        # Calculate similarity for each chunk in this document
+        for chunk_info in doc_chunks:
+            chunk_embedding = chunk_info['embedding']
+            similarity = similarity_func(query_embedding, chunk_embedding)
+            
+            chunk_similarities.append({
+                'document': doc_name,
+                'chunk_id': chunk_info['chunk_id'],
+                'text': chunk_info['text'],
+                'start_char': chunk_info.get('start_char', 0),
+                'end_char': chunk_info.get('end_char', len(chunk_info['text'])),
+                'token_count': chunk_info.get('token_count', len(chunk_info['text'].split())),
+                'similarity': similarity
+            })
+        
+        # Get top chunks from this document
+        chunk_similarities.sort(key=lambda x: x['similarity'], reverse=True)
+        top_chunks = chunk_similarities[:top_chunks_per_doc]
+        all_relevant_chunks.extend(top_chunks)
+    
+    # Sort all chunks by similarity
+    all_relevant_chunks.sort(key=lambda x: x['similarity'], reverse=True)
+    
+    print(f"🔍 Found {len(all_relevant_chunks)} relevant chunks (Top-K)")
+    for i, chunk in enumerate(all_relevant_chunks[:5]):  # Show top 5
+        print(f"  {i+1}. {chunk['document']} (chunk {chunk['chunk_id']}, similarity: {chunk['similarity']:.3f})")
+        print(f"     Preview: {chunk['text'][:100]}...")
+    
+    return all_relevant_chunks
+
+
+def find_relevant_chunks_top_p(query: str, model: SentenceTransformer,
+                              relevant_docs: List[str], chunk_embeddings: Dict,
+                              top_p: float = 0.6, min_similarity: float = 0.3,
+                              similarity_metric: str = "cosine") -> List[Dict]:
+    """Find most relevant chunks using Top-P strategy for better quality control."""
+    query_embedding = create_text_embedding(model, query)
+    
+    # Collect all chunks from all relevant documents
+    all_chunk_similarities = []
+    
+    for doc_name in relevant_docs:
+        if doc_name not in chunk_embeddings:
+            continue
+            
+        doc_chunks = chunk_embeddings[doc_name]
+        
+        # Get similarity function
+        similarity_func = SIMILARITY_FUNCTIONS.get(similarity_metric, cosine_similarity)
+        
+        # Calculate similarity for each chunk in this document
+        for chunk_info in doc_chunks:
+            chunk_embedding = chunk_info['embedding']
+            similarity = similarity_func(query_embedding, chunk_embedding)
+            
+            # Only include chunks above minimum similarity threshold
+            if similarity >= min_similarity:
+                all_chunk_similarities.append({
+                    'document': doc_name,
+                    'chunk_id': chunk_info['chunk_id'],
+                    'text': chunk_info['text'],
+                    'start_char': chunk_info.get('start_char', 0),
+                    'end_char': chunk_info.get('end_char', len(chunk_info['text'])),
+                    'token_count': chunk_info.get('token_count', len(chunk_info['text'].split())),
+                    'similarity': similarity
+                })
+    
+    if not all_chunk_similarities:
+        print(f"⚠️ No chunks found above similarity threshold {min_similarity}")
+        return []
+    
+    # Sort by similarity
+    all_chunk_similarities.sort(key=lambda x: x['similarity'], reverse=True)
+    
+    # Apply Top-P selection
+    total_score = sum(chunk['similarity'] for chunk in all_chunk_similarities)
+    cumulative_prob = 0.0
+    selected_chunks = []
+    
+    for chunk in all_chunk_similarities:
+        prob = chunk['similarity'] / total_score
+        cumulative_prob += prob
+        selected_chunks.append(chunk)
+        
+        # Stop when we reach the Top-P threshold
+        if cumulative_prob >= top_p:
+            break
+    
+    print(f"🔍 Found {len(selected_chunks)} relevant chunks (Top-P={top_p})")
+    print(f"📊 Filtered from {len(all_chunk_similarities)} chunks above threshold")
+    print(f"📊 Cumulative probability: {cumulative_prob:.3f}")
+    
+    for i, chunk in enumerate(selected_chunks[:5]):  # Show top 5
+        print(f"  {i+1}. {chunk['document']} (chunk {chunk['chunk_id']}, similarity: {chunk['similarity']:.3f})")
+        print(f"     Preview: {chunk['text'][:100]}...")
+    
+    return selected_chunks
+
+
+def find_relevant_chunks(query: str, model: SentenceTransformer, 
+                        relevant_docs: List[str], chunk_embeddings: Dict,
+                        strategy: str = "top_p", **kwargs) -> List[Dict]:
+    """Unified interface for chunk retrieval with different strategies."""
+    
+    similarity_metric = kwargs.get("similarity_metric", "cosine")
+    
+    if strategy == "top_k":
+        top_chunks_per_doc = kwargs.get("top_chunks_per_doc", 3)
+        return find_relevant_chunks_top_k(query, model, relevant_docs, chunk_embeddings, 
+                                        top_chunks_per_doc, similarity_metric)
+    
+    elif strategy == "top_p":
+        top_p = kwargs.get("top_p", 0.6)
+        min_similarity = kwargs.get("min_similarity", 0.3)
+        return find_relevant_chunks_top_p(query, model, relevant_docs, chunk_embeddings, 
+                                        top_p, min_similarity, similarity_metric)
+    
+    else:
+        raise ValueError(f"Unknown strategy: {strategy}. Use 'top_k' or 'top_p'")
+
+
+def get_documents_for_rag(relevant_docs: List[str], document_index: Dict) -> List[str]:
+    """Get full content of relevant documents for RAG processing."""
+    rag_documents = []
+    
+    for doc_name in relevant_docs:
+        if doc_name in document_index:
+            content = document_index[doc_name].get('full_content', document_index[doc_name].get('content', ''))
+            if content.strip():
+                rag_documents.append(content)
+    
+    print(f"📄 Retrieved {len(rag_documents)} documents for RAG")
+    return rag_documents
+
+
+def get_chunks_for_rag(relevant_chunks: List[Dict], max_chunks: int = 10) -> List[str]:
+    """Get the most relevant chunks for RAG processing."""
+    # Take top chunks and format them with context
+    selected_chunks = relevant_chunks[:max_chunks]
+    
+    rag_chunks = []
+    for chunk in selected_chunks:
+        formatted_chunk = f"[Document: {chunk['document']}, Chunk {chunk['chunk_id']}]\n{chunk['text']}"
+        rag_chunks.append(formatted_chunk)
+    
+    print(f"📄 Retrieved {len(rag_chunks)} chunks for RAG")
+    return rag_chunks

src/pdf-version/retrieval/document_retriever.py

@@ -0,0 +1,192 @@
+"""Document retrieval strategies and functionality."""
+
+from typing import List, Dict
+import numpy as np
+from sentence_transformers import SentenceTransformer
+from src.indexing.embedding_creator import create_text_embedding
+
+
+def find_relevant_documents_top_k(query: str, model: SentenceTransformer, 
+                                tag_embeddings: Dict, doc_tag_mapping: Dict, 
+                                top_k: int = 3) -> List[str]:
+    """Find top-k most relevant documents based on query similarity to tags."""
+    query_embedding = create_text_embedding(model, query)
+    
+    # Calculate similarity between query and all tags
+    tag_similarities = {}
+    for tag, tag_embedding in tag_embeddings.items():
+        similarity = np.dot(query_embedding, tag_embedding) / (
+            np.linalg.norm(query_embedding) * np.linalg.norm(tag_embedding)
+        )
+        tag_similarities[tag] = similarity
+    
+    # Find documents that contain the most similar tags
+    doc_scores = {}
+    for pdf_name, doc_info in doc_tag_mapping.items():
+        doc_tags = doc_info['tags']
+        
+        # Calculate document score using max similarity for precise tag matching
+        if doc_tags:
+            similarities = [tag_similarities.get(tag, 0) for tag in doc_tags]
+            # Use max similarity to find documents with best tag matches
+            doc_score = max(similarities)
+            doc_scores[pdf_name] = doc_score
+    
+    # Sort and return top-k documents
+    sorted_docs = sorted(doc_scores.items(), key=lambda x: x[1], reverse=True)
+    relevant_docs = [doc_name for doc_name, score in sorted_docs[:top_k]]
+    
+    print(f"🔍 Found {len(relevant_docs)} relevant documents for query: '{query}' (TOP-K)")
+    for i, doc_name in enumerate(relevant_docs):
+        score = doc_scores[doc_name]
+        print(f"  {i+1}. {doc_name} (similarity: {score:.3f})")
+    
+    return relevant_docs
+
+
+def find_relevant_documents_top_p(query: str, model: SentenceTransformer, 
+                                tag_embeddings: Dict, doc_tag_mapping: Dict, 
+                                top_p: float = 0.6, min_similarity: float = 0.5) -> List[str]:
+    """Find documents using TOP-P (nucleus sampling) approach."""
+    query_embedding = create_text_embedding(model, query)
+    
+    # Calculate similarity between query and all tags
+    tag_similarities = {}
+    for tag, tag_embedding in tag_embeddings.items():
+        similarity = np.dot(query_embedding, tag_embedding) / (
+            np.linalg.norm(query_embedding) * np.linalg.norm(tag_embedding)
+        )
+        tag_similarities[tag] = similarity
+    
+    # Find documents that contain the most similar tags
+    doc_scores = {}
+    for pdf_name, doc_info in doc_tag_mapping.items():
+        doc_tags = doc_info['tags']
+        
+        # Calculate document score using max similarity for precise tag matching
+        if doc_tags:
+            similarities = [tag_similarities.get(tag, 0) for tag in doc_tags]
+            # Use max similarity to find documents with best tag matches
+            doc_score = max(similarities)
+            doc_scores[pdf_name] = doc_score
+    
+    # Filter out documents below minimum similarity threshold
+    filtered_docs = {doc: score for doc, score in doc_scores.items() 
+                    if score >= min_similarity}
+    
+    if not filtered_docs:
+        print(f"⚠️ No documents found above similarity threshold {min_similarity}")
+        return []
+    
+    # Sort documents by similarity score
+    sorted_docs = sorted(filtered_docs.items(), key=lambda x: x[1], reverse=True)
+    
+    # Apply TOP-P selection
+    total_score = sum(score for _, score in sorted_docs)
+    cumulative_prob = 0.0
+    selected_docs = []
+    
+    for doc_name, score in sorted_docs:
+        prob = score / total_score
+        cumulative_prob += prob
+        selected_docs.append(doc_name)
+        
+        # Stop when we reach the TOP-P threshold
+        if cumulative_prob >= top_p:
+            break
+    
+    print(f"🔍 Found {len(selected_docs)} relevant documents for query: '{query}' (TOP-P={top_p})")
+    print(f"📊 Cumulative probability: {cumulative_prob:.3f}")
+    
+    for i, doc_name in enumerate(selected_docs):
+        score = doc_scores[doc_name]
+        prob = score / total_score
+        print(f"  {i+1}. {doc_name} (similarity: {score:.3f}, prob: {prob:.3f})")
+    
+    return selected_docs
+
+
+def find_relevant_documents_threshold(query: str, model: SentenceTransformer, 
+                                    tag_embeddings: Dict, doc_tag_mapping: Dict, 
+                                    similarity_threshold: float = 0.5) -> List[str]:
+    """Find all documents above a similarity threshold."""
+    query_embedding = create_text_embedding(model, query)
+    
+    # Calculate similarity between query and all tags
+    tag_similarities = {}
+    for tag, tag_embedding in tag_embeddings.items():
+        similarity = np.dot(query_embedding, tag_embedding) / (
+            np.linalg.norm(query_embedding) * np.linalg.norm(tag_embedding)
+        )
+        tag_similarities[tag] = similarity
+    
+    # Find documents that contain the most similar tags
+    doc_scores = {}
+    for pdf_name, doc_info in doc_tag_mapping.items():
+        doc_tags = doc_info['tags']
+        
+        # Calculate document score using weighted average
+        if doc_tags:
+            similarities = [tag_similarities.get(tag, 0) for tag in doc_tags]
+            avg_similarity = np.mean(similarities)
+            max_similarity = max(similarities)
+            # Weighted combination: 70% average (overall relevance) + 30% max (strongest match)
+            doc_score = avg_similarity * 0.7 + max_similarity * 0.3
+            if doc_score >= similarity_threshold:
+                doc_scores[pdf_name] = doc_score
+    
+    # Sort by similarity score
+    sorted_docs = sorted(doc_scores.items(), key=lambda x: x[1], reverse=True)
+    relevant_docs = [doc_name for doc_name, score in sorted_docs]
+    
+    print(f"🔍 Found {len(relevant_docs)} relevant documents for query: '{query}' (threshold={similarity_threshold})")
+    for i, doc_name in enumerate(relevant_docs):
+        score = doc_scores[doc_name]
+        print(f"  {i+1}. {doc_name} (similarity: {score:.3f})")
+    
+    return relevant_docs
+
+
+def find_relevant_documents(query: str, model: SentenceTransformer, 
+                          tag_embeddings: Dict, doc_tag_mapping: Dict, 
+                          strategy: str = "top_k", **kwargs) -> List[str]:
+    """Unified interface for finding relevant documents with different strategies."""
+    if strategy == "top_k":
+        top_k = kwargs.get("top_k", 3)
+        return find_relevant_documents_top_k(query, model, tag_embeddings, doc_tag_mapping, top_k)
+    
+    elif strategy == "top_p":
+        top_p = kwargs.get("top_p", 0.6)
+        min_similarity = kwargs.get("min_similarity", 0.5)
+        return find_relevant_documents_top_p(query, model, tag_embeddings, doc_tag_mapping, top_p, min_similarity)
+    
+    elif strategy == "threshold":
+        similarity_threshold = kwargs.get("similarity_threshold", 0.5)
+        return find_relevant_documents_threshold(query, model, tag_embeddings, doc_tag_mapping, similarity_threshold)
+    
+    else:
+        raise ValueError(f"Unknown strategy: {strategy}. Use 'top_k', 'top_p', or 'threshold'")
+
+
+def create_document_tag_mapping(document_index: Dict, tag_embeddings: Dict) -> Dict:
+    """Create mapping between documents and their tag embeddings."""
+    doc_tag_mapping = {}
+    
+    for pdf_name, doc_info in document_index.items():
+        doc_tags = doc_info['all_tags']
+        
+        # Get embeddings for this document's tags
+        doc_tag_embeddings = {}
+        for tag in doc_tags:
+            if tag in tag_embeddings:
+                doc_tag_embeddings[tag] = tag_embeddings[tag]
+        
+        doc_tag_mapping[pdf_name] = {
+            'tags': doc_tags,
+            'tag_embeddings': doc_tag_embeddings,
+            'symptoms': doc_info['symptoms'],
+            'diagnoses': doc_info['diagnoses'],
+            'treatments': doc_info.get('treatments', [])
+        }
+    
+    return doc_tag_mapping

src/pdf-version/utils/__init__.py

@@ -0,0 +1,5 @@
+"""Utility functions."""
+
+from .helpers import *
+
+__all__ = []

src/pdf-version/utils/helpers.py

@@ -0,0 +1,4 @@
+"""Utility functions and helpers."""
+
+# Placeholder for utility functions
+# Add common helper functions here as needed