Add latency and relevance evaluators for medical query analysis (evaluatoin)

- Implemented LatencyEvaluator to measure processing time for individual medical queries, including detailed timing for each processing step.
- Created a pre-defined query file (pre_user_query_evaluate.txt) with sample queries categorized as diagnosis, treatment, and mixed.
- Developed RelevanceEvaluator to assess retrieval relevance using cosine similarity, with automatic evaluation based on existing similarity scores.
- Added functionality to parse queries from a file and generate comprehensive statistics for both latency and relevance evaluations.
- Included methods to save results and statistics in JSON format for further analysis and visualization.

evaluation/coverage_evaluator.py

@@ -0,0 +1,560 @@
+#!/usr/bin/env python3
+"""
+OnCall.ai System - Retrieval Coverage Evaluator (Metric 4)
+==========================================================
+
+Evaluates how well generated medical advice utilizes retrieved content
+Automatic evaluation using keyword overlap analysis with optional LLM sampling
+
+Author: YanBo Chen  
+Date: 2025-08-04
+"""
+
+import json
+import os
+import sys
+from typing import Dict, List, Any, Set
+from datetime import datetime
+from pathlib import Path
+import re
+
+# Add project path
+current_dir = Path(__file__).parent
+project_root = current_dir.parent
+src_dir = project_root / "src"
+sys.path.insert(0, str(src_dir))
+
+# Import existing system components
+try:
+    from user_prompt import UserPromptProcessor
+    from retrieval import BasicRetrievalSystem
+    from llm_clients import llm_Med42_70BClient
+    from generation import MedicalAdviceGenerator
+except ImportError as e:
+    print(f"❌ Import failed: {e}")
+    print("Please ensure running from project root directory")
+    sys.exit(1)
+
+
+class CoverageEvaluator:
+    """Retrieval coverage evaluator using keyword overlap analysis"""
+    
+    def __init__(self):
+        """Initialize system components for coverage testing"""
+        print("🔧 Initializing Coverage Evaluator...")
+        
+        # Initialize full pipeline components (needed for advice generation)
+        self.llm_client = llm_Med42_70BClient()
+        self.retrieval_system = BasicRetrievalSystem()
+        self.user_prompt_processor = UserPromptProcessor(
+            llm_client=self.llm_client,
+            retrieval_system=self.retrieval_system
+        )
+        self.medical_generator = MedicalAdviceGenerator(llm_client=self.llm_client)
+        
+        # Results accumulation
+        self.coverage_results = []
+        
+        print("✅ Coverage Evaluator initialization complete")
+    
+    def extract_medical_keywords(self, text: str) -> Set[str]:
+        """
+        Extract medical keywords from text for coverage analysis
+        
+        Uses medical terminology patterns and common medical terms
+        """
+        if not text:
+            return set()
+        
+        medical_keywords = set()
+        text_lower = text.lower()
+        
+        # Medical terminology patterns
+        patterns = [
+            r'\b[a-z]+(?:osis|itis|pathy|emia|uria|gram|scopy)\b',  # Medical suffixes
+            r'\b(?:cardio|neuro|pulmo|gastro|hepato|nephro)[a-z]+\b',  # Medical prefixes
+            r'\b(?:diagnosis|treatment|therapy|intervention|management)\b',  # Medical actions
+            r'\b(?:patient|symptom|condition|disease|disorder|syndrome)\b',  # Medical entities
+            r'\b(?:acute|chronic|severe|mild|moderate|emergency)\b',  # Medical descriptors
+            r'\b[a-z]+(?:al|ic|ous|ive)\s+(?:pain|failure|infection|injury)\b',  # Compound terms
+            r'\b(?:ecg|ekg|ct|mri|x-ray|ultrasound|biopsy)\b',  # Medical procedures
+            r'\b\d+\s*(?:mg|ml|units|hours|days|minutes)\b',  # Dosages and timeframes
+        ]
+        
+        for pattern in patterns:
+            matches = re.findall(pattern, text_lower)
+            medical_keywords.update(match.strip() for match in matches)
+        
+        # Additional common medical terms
+        common_medical_terms = [
+            'blood', 'pressure', 'heart', 'chest', 'pain', 'stroke', 'seizure',
+            'emergency', 'hospital', 'monitor', 'assess', 'evaluate', 'immediate',
+            'protocol', 'guideline', 'recommendation', 'risk', 'factor'
+        ]
+        
+        for term in common_medical_terms:
+            if term in text_lower:
+                medical_keywords.add(term)
+        
+        # Filter out very short terms and common words
+        filtered_keywords = {
+            kw for kw in medical_keywords 
+            if len(kw) > 2 and kw not in ['the', 'and', 'for', 'with', 'are', 'can', 'may']
+        }
+        
+        return filtered_keywords
+    
+    def calculate_coverage_score(self, generated_advice: str, retrieval_results: List[Dict]) -> Dict[str, Any]:
+        """
+        Calculate coverage score based on keyword overlap between advice and retrieved docs
+        
+        Args:
+            generated_advice: Generated medical advice text
+            retrieval_results: List of retrieved documents
+        """
+        if not generated_advice or not retrieval_results:
+            return {
+                "coverage_score": 0.0,
+                "matched_keywords": [],
+                "advice_keywords": [],
+                "source_keywords": [],
+                "coverage_details": []
+            }
+        
+        # Extract keywords from generated advice
+        advice_keywords = self.extract_medical_keywords(generated_advice)
+        
+        # Extract keywords from all retrieved documents
+        all_source_keywords = set()
+        coverage_details = []
+        
+        for i, doc in enumerate(retrieval_results):
+            doc_content = doc.get('content', '') or doc.get('text', '')
+            doc_keywords = self.extract_medical_keywords(doc_content)
+            all_source_keywords.update(doc_keywords)
+            
+            # Calculate overlap for this specific document
+            doc_overlap = advice_keywords.intersection(doc_keywords)
+            doc_coverage = len(doc_overlap) / len(doc_keywords) if doc_keywords else 0.0
+            
+            coverage_details.append({
+                "doc_index": i,
+                "doc_snippet": doc_content[:100] + "...",
+                "doc_keywords_count": len(doc_keywords),
+                "matched_keywords_count": len(doc_overlap),
+                "doc_coverage_ratio": doc_coverage,
+                "matched_keywords": list(doc_overlap)[:10]  # Limit for readability
+            })
+        
+        # Calculate overall coverage
+        matched_keywords = advice_keywords.intersection(all_source_keywords)
+        coverage_score = len(matched_keywords) / len(all_source_keywords) if all_source_keywords else 0.0
+        
+        return {
+            "coverage_score": coverage_score,
+            "matched_keywords": list(matched_keywords),
+            "advice_keywords": list(advice_keywords),
+            "source_keywords": list(all_source_keywords),
+            "advice_keywords_count": len(advice_keywords),
+            "source_keywords_count": len(all_source_keywords),
+            "matched_keywords_count": len(matched_keywords),
+            "coverage_percentage": coverage_score * 100,
+            "meets_threshold": coverage_score >= 0.6,
+            "coverage_details": coverage_details
+        }
+    
+    def evaluate_single_coverage(self, query: str, category: str = "unknown") -> Dict[str, Any]:
+        """
+        Evaluate retrieval coverage for a single query
+        
+        Requires full pipeline: extraction → retrieval → generation → coverage analysis
+        
+        Args:
+            query: Medical query to test
+            category: Query category (diagnosis/treatment/mixed)
+        """
+        print(f"🔍 Testing coverage for: {query[:50]}...")
+        print(f"📋 Category: {category}")
+        
+        try:
+            # Step 1: Extract condition
+            condition_result = self.user_prompt_processor.extract_condition_keywords(query)
+            
+            # Step 2: Perform retrieval
+            search_query = f"{condition_result.get('emergency_keywords', '')} {condition_result.get('treatment_keywords', '')}".strip()
+            if not search_query:
+                search_query = condition_result.get('condition', query)
+            
+            retrieval_start = datetime.now()
+            retrieval_results = self.retrieval_system.search(search_query, top_k=5)
+            retrieval_time = (datetime.now() - retrieval_start).total_seconds()
+            
+            processed_results = retrieval_results.get('processed_results', [])
+            
+            if not processed_results:
+                result = {
+                    "query": query,
+                    "category": category,
+                    "search_query": search_query,
+                    "pipeline_success": False,
+                    "coverage_score": 0.0,
+                    "error": "No retrieval results",
+                    "timestamp": datetime.now().isoformat()
+                }
+                
+                self.coverage_results.append(result)
+                print(f"   ❌ No retrieval results for coverage analysis")
+                return result
+            
+            # Step 3: Generate medical advice
+            generation_start = datetime.now()
+            intention = self._detect_query_intention(query)
+            medical_advice_result = self.medical_generator.generate_medical_advice(
+                user_query=query,
+                retrieval_results=retrieval_results,
+                intention=intention
+            )
+            generation_time = (datetime.now() - generation_start).total_seconds()
+            
+            generated_advice = medical_advice_result.get('medical_advice', '')
+            
+            if not generated_advice:
+                result = {
+                    "query": query,
+                    "category": category,
+                    "search_query": search_query,
+                    "pipeline_success": False,
+                    "coverage_score": 0.0,
+                    "error": "No generated advice",
+                    "timestamp": datetime.now().isoformat()
+                }
+                
+                self.coverage_results.append(result)
+                print(f"   ❌ No generated advice for coverage analysis")
+                return result
+            
+            # Step 4: Calculate coverage
+            coverage_analysis = self.calculate_coverage_score(generated_advice, processed_results)
+            
+            result = {
+                "query": query,
+                "category": category,
+                "search_query": search_query,
+                "pipeline_success": True,
+                "retrieval_time": retrieval_time,
+                "generation_time": generation_time,
+                "retrieved_docs_count": len(processed_results),
+                "generated_advice_length": len(generated_advice),
+                "coverage_analysis": coverage_analysis,
+                "coverage_score": coverage_analysis['coverage_score'],
+                "meets_threshold": coverage_analysis['meets_threshold'],
+                "timestamp": datetime.now().isoformat()
+            }
+            
+            # Store result
+            self.coverage_results.append(result)
+            
+            print(f"   ✅ Pipeline: Complete")
+            print(f"   📊 Coverage Score: {coverage_analysis['coverage_score']:.3f} ({coverage_analysis['coverage_percentage']:.1f}%)")
+            print(f"   📝 Keywords: {coverage_analysis['matched_keywords_count']}/{coverage_analysis['source_keywords_count']} matched")
+            print(f"   🎯 Threshold: {'✅ Met' if result['meets_threshold'] else '❌ Not Met'}")
+            print(f"   ⏱️ Times: Retrieval={retrieval_time:.2f}s, Generation={generation_time:.2f}s")
+            
+            return result
+            
+        except Exception as e:
+            error_result = {
+                "query": query,
+                "category": category,
+                "pipeline_success": False,
+                "coverage_score": 0.0,
+                "error": str(e),
+                "timestamp": datetime.now().isoformat()
+            }
+            
+            self.coverage_results.append(error_result)
+            print(f"   ❌ Coverage evaluation failed: {e}")
+            
+            return error_result
+    
+    def _detect_query_intention(self, query: str) -> str:
+        """Simplified query intention detection (from app.py)"""
+        query_lower = query.lower()
+        
+        if any(word in query_lower for word in ['diagnos', 'differential', 'possible', 'causes']):
+            return 'diagnosis'
+        elif any(word in query_lower for word in ['treat', 'manage', 'therapy', 'intervention']):
+            return 'treatment'
+        else:
+            return 'mixed'
+    
+    def parse_queries_from_file(self, filepath: str) -> Dict[str, List[Dict]]:
+        """Parse queries from file with category labels"""
+        print(f"📁 Reading queries from file: {filepath}")
+        
+        try:
+            with open(filepath, 'r', encoding='utf-8') as f:
+                content = f.read()
+            
+            # Parse queries with category labels
+            queries_by_category = {
+                "diagnosis": [],
+                "treatment": [], 
+                "mixed": []
+            }
+            
+            lines = content.strip().split('\n')
+            
+            for line in lines:
+                line = line.strip()
+                if not line:
+                    continue
+                
+                # Parse format: "1.diagnosis: query text"
+                match = re.match(r'^\d+\.(diagnosis|treatment|mixed/complicated|mixed):\s*(.+)', line, re.IGNORECASE)
+                if match:
+                    category_raw = match.group(1).lower()
+                    query_text = match.group(2).strip()
+                    
+                    # Normalize category name
+                    if category_raw in ['mixed/complicated', 'mixed']:
+                        category = 'mixed'
+                    else:
+                        category = category_raw
+                    
+                    if category in queries_by_category and len(query_text) > 15:
+                        queries_by_category[category].append({
+                            "text": query_text,
+                            "category": category
+                        })
+            
+            print(f"📋 Parsed queries by category:")
+            for category, category_queries in queries_by_category.items():
+                print(f"  {category.capitalize()}: {len(category_queries)} queries")
+            
+            return queries_by_category
+            
+        except Exception as e:
+            print(f"❌ Failed to read file: {e}")
+            return {"error": f"Failed to read file: {e}"}
+    
+    def calculate_coverage_statistics(self) -> Dict[str, Any]:
+        """Calculate coverage statistics by category"""
+        category_stats = {}
+        all_successful_results = []
+        
+        # Group results by category
+        results_by_category = {
+            "diagnosis": [],
+            "treatment": [],
+            "mixed": []
+        }
+        
+        for result in self.coverage_results:
+            category = result.get('category', 'unknown')
+            if category in results_by_category:
+                results_by_category[category].append(result)
+                if result.get('pipeline_success'):
+                    all_successful_results.append(result)
+        
+        # Calculate statistics for each category
+        for category, results in results_by_category.items():
+            successful_results = [r for r in results if r.get('pipeline_success')]
+            
+            if successful_results:
+                coverage_scores = [r['coverage_score'] for r in successful_results]
+                avg_coverage = sum(coverage_scores) / len(coverage_scores)
+                avg_retrieval_time = sum(r.get('retrieval_time', 0) for r in successful_results) / len(successful_results)
+                avg_generation_time = sum(r.get('generation_time', 0) for r in successful_results) / len(successful_results)
+                
+                category_stats[category] = {
+                    "average_coverage": avg_coverage,
+                    "max_coverage": max(coverage_scores),
+                    "min_coverage": min(coverage_scores),
+                    "successful_evaluations": len(successful_results),
+                    "total_queries": len(results),
+                    "success_rate": len(successful_results) / len(results),
+                    "average_retrieval_time": avg_retrieval_time,
+                    "average_generation_time": avg_generation_time,
+                    "meets_threshold": avg_coverage >= 0.6,
+                    "individual_coverage_scores": coverage_scores
+                }
+            else:
+                category_stats[category] = {
+                    "average_coverage": 0.0,
+                    "max_coverage": 0.0,
+                    "min_coverage": 0.0,
+                    "successful_evaluations": 0,
+                    "total_queries": len(results),
+                    "success_rate": 0.0,
+                    "average_retrieval_time": 0.0,
+                    "average_generation_time": 0.0,
+                    "meets_threshold": False,
+                    "individual_coverage_scores": []
+                }
+        
+        # Calculate overall statistics
+        if all_successful_results:
+            all_coverage_scores = [r['coverage_score'] for r in all_successful_results]
+            overall_stats = {
+                "average_coverage": sum(all_coverage_scores) / len(all_coverage_scores),
+                "max_coverage": max(all_coverage_scores),
+                "min_coverage": min(all_coverage_scores),
+                "successful_evaluations": len(all_successful_results),
+                "total_queries": len(self.coverage_results),
+                "success_rate": len(all_successful_results) / len(self.coverage_results),
+                "meets_threshold": (sum(all_coverage_scores) / len(all_coverage_scores)) >= 0.6,
+                "target_compliance": (sum(all_coverage_scores) / len(all_coverage_scores)) >= 0.6
+            }
+        else:
+            overall_stats = {
+                "average_coverage": 0.0,
+                "max_coverage": 0.0,
+                "min_coverage": 0.0,
+                "successful_evaluations": 0,
+                "total_queries": len(self.coverage_results),
+                "success_rate": 0.0,
+                "meets_threshold": False,
+                "target_compliance": False
+            }
+        
+        return {
+            "category_results": category_stats,
+            "overall_results": overall_stats,
+            "timestamp": datetime.now().isoformat()
+        }
+    
+    def save_coverage_statistics(self, filename: str = None) -> str:
+        """Save coverage statistics for chart generation"""
+        stats = self.calculate_coverage_statistics()
+        
+        if filename is None:
+            timestamp = datetime.now().strftime("%Y%m%d_%H%M%S")
+            filename = f"coverage_statistics_{timestamp}.json"
+        
+        # Ensure results directory exists
+        results_dir = Path(__file__).parent / "results"
+        results_dir.mkdir(exist_ok=True)
+        
+        filepath = results_dir / filename
+        
+        with open(filepath, 'w', encoding='utf-8') as f:
+            json.dump(stats, f, indent=2, ensure_ascii=False)
+        
+        print(f"📊 Coverage statistics saved to: {filepath}")
+        return str(filepath)
+    
+    def save_coverage_details(self, filename: str = None) -> str:
+        """Save detailed coverage results"""
+        if filename is None:
+            timestamp = datetime.now().strftime("%Y%m%d_%H%M%S")
+            filename = f"coverage_details_{timestamp}.json"
+        
+        # Ensure results directory exists
+        results_dir = Path(__file__).parent / "results"
+        results_dir.mkdir(exist_ok=True)
+        
+        filepath = results_dir / filename
+        
+        # Create comprehensive coverage data
+        coverage_data = {
+            "evaluation_metadata": {
+                "total_queries": len(self.coverage_results),
+                "successful_evaluations": len([r for r in self.coverage_results if r.get('pipeline_success')]),
+                "timestamp": datetime.now().isoformat(),
+                "evaluator_type": "retrieval_coverage",
+                "threshold_used": 0.6
+            },
+            "coverage_results": self.coverage_results
+        }
+        
+        with open(filepath, 'w', encoding='utf-8') as f:
+            json.dump(coverage_data, f, indent=2, ensure_ascii=False)
+        
+        print(f"📝 Coverage details saved to: {filepath}")
+        return str(filepath)
+
+
+# Independent execution interface
+if __name__ == "__main__":
+    """Independent coverage evaluation interface"""
+    
+    print("📈 OnCall.ai Coverage Evaluator - Retrieval Coverage Analysis")
+    
+    if len(sys.argv) > 1:
+        query_file = sys.argv[1]
+    else:
+        # Default to evaluation/pre_user_query_evaluate.txt
+        query_file = Path(__file__).parent / "pre_user_query_evaluate.txt"
+    
+    if not os.path.exists(query_file):
+        print(f"❌ Query file not found: {query_file}")
+        print("Usage: python coverage_evaluator.py [query_file.txt]")
+        sys.exit(1)
+    
+    # Initialize evaluator
+    evaluator = CoverageEvaluator()
+    
+    # Parse queries from file
+    queries_by_category = evaluator.parse_queries_from_file(str(query_file))
+    
+    if "error" in queries_by_category:
+        print(f"❌ Failed to parse queries: {queries_by_category['error']}")
+        sys.exit(1)
+    
+    # Test coverage for each query (requires full pipeline)
+    print(f"\n🧪 Retrieval Coverage Testing (Full Pipeline Required)")
+    print(f"⚠️ Note: This evaluator requires LLM calls for advice generation")
+    
+    for category, queries in queries_by_category.items():
+        if not queries:
+            continue
+            
+        print(f"\n📂 Testing {category.upper()} coverage:")
+        
+        for i, query_info in enumerate(queries):
+            query_text = query_info['text']
+            
+            # Test coverage (requires full pipeline)
+            result = evaluator.evaluate_single_coverage(query_text, category)
+            
+            # Pause between queries to avoid rate limits
+            if i < len(queries) - 1:
+                print(f"   ⏳ Pausing 5s before next query...")
+                import time
+                time.sleep(5)
+        
+        # Longer pause between categories
+        if category != list(queries_by_category.keys())[-1]:
+            print(f"\n⏳ Pausing 10s before next category...")
+            import time
+            time.sleep(10)
+    
+    # Generate and save results
+    print(f"\n📊 Generating coverage analysis...")
+    
+    # Save statistics and details
+    stats_path = evaluator.save_coverage_statistics()
+    details_path = evaluator.save_coverage_details()
+    
+    # Print final summary
+    stats = evaluator.calculate_coverage_statistics()
+    category_results = stats['category_results']
+    overall_results = stats['overall_results']
+    
+    print(f"\n📊 === COVERAGE EVALUATION SUMMARY ===")
+    print(f"Overall Performance:")
+    print(f"   Average Coverage: {overall_results['average_coverage']:.3f} ({overall_results['average_coverage']*100:.1f}%)")
+    print(f"   Pipeline Success Rate: {overall_results['success_rate']:.1%}")
+    print(f"   60% Threshold: {'✅ Met' if overall_results['meets_threshold'] else '❌ Not Met'}")
+    
+    print(f"\nCategory Breakdown:")
+    for category, cat_stats in category_results.items():
+        if cat_stats['total_queries'] > 0:
+            print(f"   {category.capitalize()}: {cat_stats['average_coverage']:.3f} "
+                  f"({cat_stats['successful_evaluations']}/{cat_stats['total_queries']}) "
+                  f"[R:{cat_stats['average_retrieval_time']:.2f}s, G:{cat_stats['average_generation_time']:.2f}s]")
+    
+    print(f"\n✅ Coverage evaluation complete!")
+    print(f"📊 Statistics: {stats_path}")
+    print(f"📝 Details: {details_path}")

evaluation/extraction_evaluator.py

@@ -0,0 +1,379 @@
+#!/usr/bin/env python3
+"""
+OnCall.ai System - Condition Extraction Evaluator (Metric 2)
+============================================================
+
+Evaluates condition extraction success rate from user_prompt.py
+Pure automatic evaluation based on extract_condition_keywords() results
+
+Author: YanBo Chen  
+Date: 2025-08-04
+"""
+
+import json
+import os
+import sys
+from typing import Dict, List, Any
+from datetime import datetime
+from pathlib import Path
+import re
+
+# Add project path
+current_dir = Path(__file__).parent
+project_root = current_dir.parent
+src_dir = project_root / "src"
+sys.path.insert(0, str(src_dir))
+
+# Import existing system components
+try:
+    from user_prompt import UserPromptProcessor
+    from retrieval import BasicRetrievalSystem
+    from llm_clients import llm_Med42_70BClient
+except ImportError as e:
+    print(f"❌ Import failed: {e}")
+    print("Please ensure running from project root directory")
+    sys.exit(1)
+
+
+class ExtractionEvaluator:
+    """Condition extraction success rate evaluator - pure automatic evaluation"""
+    
+    def __init__(self):
+        """Initialize system components for extraction testing"""
+        print("🔧 Initializing Extraction Evaluator...")
+        
+        # Initialize required components for extraction
+        self.llm_client = llm_Med42_70BClient()
+        self.retrieval_system = BasicRetrievalSystem()
+        self.user_prompt_processor = UserPromptProcessor(
+            llm_client=self.llm_client,
+            retrieval_system=self.retrieval_system
+        )
+        
+        # Results accumulation
+        self.extraction_results = []
+        
+        print("✅ Extraction Evaluator initialization complete")
+    
+    def evaluate_single_extraction(self, query: str, category: str = "unknown") -> Dict[str, Any]:
+        """
+        Evaluate condition extraction success for a single query
+        
+        Tests user_prompt.py extract_condition_keywords() method
+        
+        Args:
+            query: Medical query to test
+            category: Query category (diagnosis/treatment/mixed)
+        """
+        print(f"🔍 Testing extraction for: {query[:50]}...")
+        print(f"📋 Category: {category}")
+        
+        try:
+            # Call the actual extraction method from user_prompt.py
+            extraction_start = datetime.now()
+            condition_result = self.user_prompt_processor.extract_condition_keywords(query)
+            extraction_time = (datetime.now() - extraction_start).total_seconds()
+            
+            # Analyze extraction success
+            extracted_condition = condition_result.get('condition')
+            query_status = condition_result.get('query_status')
+            emergency_keywords = condition_result.get('emergency_keywords', [])
+            treatment_keywords = condition_result.get('treatment_keywords', [])
+            fallback_level = condition_result.get('fallback_level', 'unknown')
+            
+            # Define success criteria
+            is_successful = (
+                extracted_condition and 
+                extracted_condition.strip() and 
+                extracted_condition != "unknown" and
+                query_status not in ['invalid_query', 'non_medical']
+            )
+            
+            result = {
+                "query": query,
+                "category": category,
+                "extraction_success": is_successful,
+                "extraction_time": extraction_time,
+                "extracted_condition": extracted_condition,
+                "query_status": query_status,
+                "emergency_keywords": emergency_keywords,
+                "treatment_keywords": treatment_keywords,
+                "fallback_level": fallback_level,
+                "full_condition_result": condition_result,
+                "timestamp": datetime.now().isoformat()
+            }
+            
+            # Store result
+            self.extraction_results.append(result)
+            
+            print(f"   ✅ Extraction: {'Success' if is_successful else 'Failed'}")
+            print(f"   📝 Condition: {extracted_condition}")
+            print(f"   🎯 Status: {query_status}")
+            print(f"   ⏱️ Time: {extraction_time:.3f}s")
+            print(f"   🔄 Fallback Level: {fallback_level}")
+            
+            return result
+            
+        except Exception as e:
+            error_result = {
+                "query": query,
+                "category": category,
+                "extraction_success": False,
+                "extraction_time": 0.0,
+                "error": str(e),
+                "timestamp": datetime.now().isoformat()
+            }
+            
+            self.extraction_results.append(error_result)
+            print(f"   ❌ Extraction failed: {e}")
+            
+            return error_result
+    
+    def parse_queries_from_file(self, filepath: str) -> Dict[str, List[Dict]]:
+        """Parse queries from file with category labels"""
+        print(f"📁 Reading queries from file: {filepath}")
+        
+        try:
+            with open(filepath, 'r', encoding='utf-8') as f:
+                content = f.read()
+            
+            # Parse queries with category labels
+            queries_by_category = {
+                "diagnosis": [],
+                "treatment": [], 
+                "mixed": []
+            }
+            
+            lines = content.strip().split('\n')
+            
+            for line in lines:
+                line = line.strip()
+                if not line:
+                    continue
+                
+                # Parse format: "1.diagnosis: query text"
+                match = re.match(r'^\d+\.(diagnosis|treatment|mixed/complicated|mixed):\s*(.+)', line, re.IGNORECASE)
+                if match:
+                    category_raw = match.group(1).lower()
+                    query_text = match.group(2).strip()
+                    
+                    # Normalize category name
+                    if category_raw in ['mixed/complicated', 'mixed']:
+                        category = 'mixed'
+                    else:
+                        category = category_raw
+                    
+                    if category in queries_by_category and len(query_text) > 15:
+                        queries_by_category[category].append({
+                            "text": query_text,
+                            "category": category
+                        })
+            
+            print(f"📋 Parsed queries by category:")
+            for category, category_queries in queries_by_category.items():
+                print(f"  {category.capitalize()}: {len(category_queries)} queries")
+            
+            return queries_by_category
+            
+        except Exception as e:
+            print(f"❌ Failed to read file: {e}")
+            return {"error": f"Failed to read file: {e}"}
+    
+    def calculate_extraction_statistics(self) -> Dict[str, Any]:
+        """Calculate extraction success statistics by category"""
+        category_stats = {}
+        all_results = []
+        
+        # Group results by category
+        results_by_category = {
+            "diagnosis": [],
+            "treatment": [],
+            "mixed": []
+        }
+        
+        for result in self.extraction_results:
+            category = result.get('category', 'unknown')
+            if category in results_by_category:
+                results_by_category[category].append(result)
+                all_results.append(result)
+        
+        # Calculate statistics for each category
+        for category, results in results_by_category.items():
+            if results:
+                successful = [r for r in results if r.get('extraction_success')]
+                success_rate = len(successful) / len(results)
+                avg_time = sum(r.get('extraction_time', 0) for r in results) / len(results)
+                
+                category_stats[category] = {
+                    "success_rate": success_rate,
+                    "successful_count": len(successful),
+                    "total_count": len(results),
+                    "average_extraction_time": avg_time,
+                    "fallback_levels": [r.get('fallback_level') for r in results]
+                }
+            else:
+                category_stats[category] = {
+                    "success_rate": 0.0,
+                    "successful_count": 0,
+                    "total_count": 0,
+                    "average_extraction_time": 0.0,
+                    "fallback_levels": []
+                }
+        
+        # Calculate overall statistics
+        if all_results:
+            overall_successful = [r for r in all_results if r.get('extraction_success')]
+            overall_stats = {
+                "success_rate": len(overall_successful) / len(all_results),
+                "successful_count": len(overall_successful),
+                "total_count": len(all_results),
+                "average_extraction_time": sum(r.get('extraction_time', 0) for r in all_results) / len(all_results),
+                "target_compliance": len(overall_successful) / len(all_results) >= 0.8
+            }
+        else:
+            overall_stats = {
+                "success_rate": 0.0,
+                "successful_count": 0,
+                "total_count": 0,
+                "average_extraction_time": 0.0,
+                "target_compliance": False
+            }
+        
+        return {
+            "category_results": category_stats,
+            "overall_results": overall_stats,
+            "timestamp": datetime.now().isoformat()
+        }
+    
+    def save_extraction_statistics(self, filename: str = None) -> str:
+        """Save extraction statistics for chart generation"""
+        stats = self.calculate_extraction_statistics()
+        
+        if filename is None:
+            timestamp = datetime.now().strftime("%Y%m%d_%H%M%S")
+            filename = f"extraction_statistics_{timestamp}.json"
+        
+        # Ensure results directory exists
+        results_dir = Path(__file__).parent / "results"
+        results_dir.mkdir(exist_ok=True)
+        
+        filepath = results_dir / filename
+        
+        with open(filepath, 'w', encoding='utf-8') as f:
+            json.dump(stats, f, indent=2, ensure_ascii=False)
+        
+        print(f"📊 Extraction statistics saved to: {filepath}")
+        return str(filepath)
+    
+    def save_extraction_details(self, filename: str = None) -> str:
+        """Save detailed extraction results"""
+        if filename is None:
+            timestamp = datetime.now().strftime("%Y%m%d_%H%M%S")
+            filename = f"extraction_details_{timestamp}.json"
+        
+        # Ensure results directory exists
+        results_dir = Path(__file__).parent / "results"
+        results_dir.mkdir(exist_ok=True)
+        
+        filepath = results_dir / filename
+        
+        # Create comprehensive extraction data
+        extraction_data = {
+            "evaluation_metadata": {
+                "total_queries": len(self.extraction_results),
+                "timestamp": datetime.now().isoformat(),
+                "evaluator_type": "condition_extraction"
+            },
+            "extraction_results": self.extraction_results
+        }
+        
+        with open(filepath, 'w', encoding='utf-8') as f:
+            json.dump(extraction_data, f, indent=2, ensure_ascii=False)
+        
+        print(f"📝 Extraction details saved to: {filepath}")
+        return str(filepath)
+
+
+# Independent execution interface
+if __name__ == "__main__":
+    """Independent extraction evaluation interface"""
+    
+    print("🔍 OnCall.ai Extraction Evaluator - Condition Extraction Success Rate")
+    
+    if len(sys.argv) > 1:
+        query_file = sys.argv[1]
+    else:
+        # Default to evaluation/pre_user_query_evaluate.txt
+        query_file = Path(__file__).parent / "pre_user_query_evaluate.txt"
+    
+    if not os.path.exists(query_file):
+        print(f"❌ Query file not found: {query_file}")
+        print("Usage: python extraction_evaluator.py [query_file.txt]")
+        sys.exit(1)
+    
+    # Initialize evaluator
+    evaluator = ExtractionEvaluator()
+    
+    # Parse queries from file
+    queries_by_category = evaluator.parse_queries_from_file(str(query_file))
+    
+    if "error" in queries_by_category:
+        print(f"❌ Failed to parse queries: {queries_by_category['error']}")
+        sys.exit(1)
+    
+    # Test extraction for each query
+    print(f"\n🧪 Condition Extraction Testing")
+    
+    for category, queries in queries_by_category.items():
+        if not queries:
+            continue
+            
+        print(f"\n📂 Testing {category.upper()} extraction:")
+        
+        for i, query_info in enumerate(queries):
+            query_text = query_info['text']
+            
+            # Test extraction
+            result = evaluator.evaluate_single_extraction(query_text, category)
+            
+            # Pause between queries to avoid rate limits (if needed)
+            if i < len(queries) - 1:
+                print(f"   ⏳ Pausing 3s before next query...")
+                import time
+                time.sleep(3)
+        
+        # Pause between categories
+        if category != list(queries_by_category.keys())[-1]:
+            print(f"\n⏳ Pausing 5s before next category...")
+            import time
+            time.sleep(5)
+    
+    # Generate and save results
+    print(f"\n📊 Generating extraction analysis...")
+    
+    # Save statistics and details
+    stats_path = evaluator.save_extraction_statistics()
+    details_path = evaluator.save_extraction_details()
+    
+    # Print final summary
+    stats = evaluator.calculate_extraction_statistics()
+    category_results = stats['category_results']
+    overall_results = stats['overall_results']
+    
+    print(f"\n📊 === EXTRACTION EVALUATION SUMMARY ===")
+    print(f"Overall Performance:")
+    print(f"   Success Rate: {overall_results['success_rate']:.1%}")
+    print(f"   Successful Extractions: {overall_results['successful_count']}/{overall_results['total_count']}")
+    print(f"   Average Extraction Time: {overall_results['average_extraction_time']:.3f}s")
+    print(f"   80% Target Compliance: {'✅ Met' if overall_results['target_compliance'] else '❌ Not Met'}")
+    
+    print(f"\nCategory Breakdown:")
+    for category, cat_stats in category_results.items():
+        if cat_stats['total_count'] > 0:
+            print(f"   {category.capitalize()}: {cat_stats['success_rate']:.1%} "
+                  f"({cat_stats['successful_count']}/{cat_stats['total_count']}) "
+                  f"[{cat_stats['average_extraction_time']:.3f}s avg]")
+    
+    print(f"\n✅ Extraction evaluation complete!")
+    print(f"📊 Statistics: {stats_path}")
+    print(f"📝 Details: {details_path}")

evaluation/latency_chart_generator.py

@@ -0,0 +1,327 @@
+#!/usr/bin/env python3
+"""
+OnCall.ai System - Latency Chart Generator
+==========================================
+
+Generates comprehensive latency analysis charts from saved statistics.
+Reads JSON files produced by latency_evaluator.py and creates visualizations.
+
+No LLM calls - pure data visualization.
+
+Author: YanBo Chen  
+Date: 2025-08-04
+"""
+
+import json
+import os
+import sys
+from typing import Dict, List, Any
+from datetime import datetime
+from pathlib import Path
+import glob
+
+# Visualization imports
+import matplotlib.pyplot as plt
+import seaborn as sns
+import pandas as pd
+import numpy as np
+
+
+class LatencyChartGenerator:
+    """Generate charts from latency evaluation statistics - no LLM dependency"""
+    
+    def __init__(self):
+        """Initialize chart generator"""
+        print("📈 Initializing Latency Chart Generator...")
+        
+        # Set up professional chart style
+        plt.style.use('default')
+        sns.set_palette("husl")
+        
+        print("✅ Chart Generator ready")
+    
+    def load_latest_statistics(self, results_dir: str = None) -> Dict[str, Any]:
+        """
+        Load the most recent latency statistics file
+        
+        Args:
+            results_dir: Directory containing statistics files
+        """
+        if results_dir is None:
+            results_dir = Path(__file__).parent / "results"
+        
+        # Find latest statistics file
+        pattern = str(results_dir / "latency_statistics_*.json")
+        stat_files = glob.glob(pattern)
+        
+        if not stat_files:
+            raise FileNotFoundError(f"No latency statistics files found in {results_dir}")
+        
+        # Get the most recent file
+        latest_file = max(stat_files, key=os.path.getmtime)
+        
+        print(f"📊 Loading statistics from: {latest_file}")
+        
+        with open(latest_file, 'r', encoding='utf-8') as f:
+            stats = json.load(f)
+        
+        return stats
+    
+    def generate_comprehensive_charts(self, stats: Dict[str, Any]) -> str:
+        """
+        Generate comprehensive 4-category latency analysis charts
+        
+        Creates professional charts showing:
+        1. Category comparison bar chart
+        2. Individual query scatter plot  
+        3. Statistical summary table
+        4. Performance distribution box plot
+        """
+        try:
+            # Create figure with subplots
+            fig, axes = plt.subplots(2, 2, figsize=(16, 12))
+            fig.suptitle('OnCall.ai Latency Analysis - Category Comparison', 
+                        fontsize=16, fontweight='bold')
+            
+            category_results = stats['category_results']
+            overall_results = stats['overall_results']
+            
+            # Chart 1: Category Comparison Bar Chart
+            ax1 = axes[0, 0]
+            categories = []
+            avg_latencies = []
+            std_devs = []
+            
+            # Collect category data
+            for category, cat_stats in category_results.items():
+                if cat_stats['query_count'] > 0:
+                    categories.append(category.replace('_', ' ').title())
+                    avg_latencies.append(cat_stats['average_latency'])
+                    std_devs.append(cat_stats['std_deviation'])
+            
+            # Add overall
+            categories.append('Overall')
+            avg_latencies.append(overall_results['average_latency'])
+            std_devs.append(overall_results['std_deviation'])
+            
+            # Create bar chart with error bars
+            bars = ax1.bar(categories, avg_latencies, capsize=5, alpha=0.8, 
+                          color=['#1f77b4', '#ff7f0e', '#d62728', '#2ca02c'])
+            ax1.errorbar(categories, avg_latencies, yerr=std_devs, fmt='none', 
+                        color='black', capsize=3, capthick=1)
+            
+            ax1.set_title('Average Latency by Category', fontweight='bold')
+            ax1.set_ylabel('Latency (seconds)')
+            ax1.set_xlabel('Query Category')
+            ax1.grid(True, alpha=0.3)
+            
+            # Add value labels on bars
+            for bar, avg, std in zip(bars, avg_latencies, std_devs):
+                height = bar.get_height()
+                ax1.text(bar.get_x() + bar.get_width()/2., height + std*0.1,
+                        f'{avg:.1f}s', ha='center', va='bottom', fontweight='bold')
+            
+            # Add target line
+            ax1.axhline(y=30.0, color='red', linestyle='--', alpha=0.7, label='30s Target')
+            ax1.legend()
+            
+            # Chart 2: Individual Query Performance
+            ax2 = axes[0, 1]
+            
+            query_indices = []
+            latencies = []
+            colors = []
+            
+            color_map = {'diagnosis': '#1f77b4', 'treatment': '#ff7f0e', 'mixed': '#d62728'}
+            query_idx = 0
+            
+            for category, cat_stats in category_results.items():
+                for latency in cat_stats['individual_latencies']:
+                    query_indices.append(query_idx)
+                    latencies.append(latency)
+                    colors.append(color_map.get(category, 'gray'))
+                    query_idx += 1
+            
+            if latencies:
+                ax2.scatter(query_indices, latencies, c=colors, alpha=0.7, s=100)
+                ax2.set_title('Individual Query Performance', fontweight='bold')
+                ax2.set_ylabel('Latency (seconds)')
+                ax2.set_xlabel('Query Index')
+                ax2.grid(True, alpha=0.3)
+                
+                # Add target line
+                ax2.axhline(y=30.0, color='red', linestyle='--', alpha=0.7, label='30s Target')
+                
+                # Add category legend
+                from matplotlib.patches import Patch
+                legend_elements = [Patch(facecolor=color_map[cat], label=cat.title()) 
+                                 for cat in color_map.keys() if cat in category_results.keys()]
+                ax2.legend(handles=legend_elements)
+            else:
+                ax2.text(0.5, 0.5, 'No latency data available', 
+                        ha='center', va='center', transform=ax2.transAxes)
+                ax2.set_title('Individual Query Performance', fontweight='bold')
+            
+            # Chart 3: Statistical Summary Table
+            ax3 = axes[1, 0]
+            ax3.axis('tight')
+            ax3.axis('off')
+            
+            # Create summary table
+            table_data = []
+            headers = ['Category', 'Avg (s)', 'Std (s)', 'Min (s)', 'Max (s)', 'Count']
+            
+            for category, cat_stats in category_results.items():
+                if cat_stats['query_count'] > 0:
+                    table_data.append([
+                        category.replace('_', ' ').title(),
+                        f"{cat_stats['average_latency']:.2f}",
+                        f"{cat_stats['std_deviation']:.2f}",
+                        f"{cat_stats['min_latency']:.2f}",
+                        f"{cat_stats['max_latency']:.2f}",
+                        str(cat_stats['query_count'])
+                    ])
+            
+            # Add overall row
+            table_data.append([
+                'Overall',
+                f"{overall_results['average_latency']:.2f}",
+                f"{overall_results['std_deviation']:.2f}",
+                f"{overall_results['min_latency']:.2f}",
+                f"{overall_results['max_latency']:.2f}",
+                str(overall_results['successful_queries'])
+            ])
+            
+            if table_data:
+                table = ax3.table(cellText=table_data, colLabels=headers,
+                                cellLoc='center', loc='center',
+                                colWidths=[0.2, 0.15, 0.15, 0.15, 0.15, 0.1])
+                table.auto_set_font_size(False)
+                table.set_fontsize(10)
+                table.scale(1, 2)
+                
+                # Style the table header
+                for i in range(len(headers)):
+                    table[(0, i)].set_text_props(weight='bold', color='white')
+                    table[(0, i)].set_facecolor('#2E7D32')
+            
+            ax3.set_title('Statistical Summary', fontweight='bold', pad=20)
+            
+            # Chart 4: Performance Distribution
+            ax4 = axes[1, 1]
+            
+            # Create box plot if we have multiple data points
+            box_data = []
+            box_labels = []
+            
+            for category, cat_stats in category_results.items():
+                if cat_stats['individual_latencies'] and len(cat_stats['individual_latencies']) > 0:
+                    box_data.append(cat_stats['individual_latencies'])
+                    box_labels.append(category.replace('_', ' ').title())
+            
+            if box_data and len(box_data) > 0:
+                box_plot = ax4.boxplot(box_data, labels=box_labels, patch_artist=True)
+                
+                # Color the boxes
+                colors = ['#1f77b4', '#ff7f0e', '#d62728']
+                for patch, color in zip(box_plot['boxes'], colors[:len(box_plot['boxes'])]):
+                    patch.set_facecolor(color)
+                    patch.set_alpha(0.7)
+                
+                ax4.set_title('Latency Distribution by Category', fontweight='bold')
+                ax4.set_ylabel('Latency (seconds)')
+                ax4.grid(True, alpha=0.3)
+                
+                # Add target line
+                ax4.axhline(y=30.0, color='red', linestyle='--', alpha=0.7, label='30s Target')
+                ax4.legend()
+            else:
+                # For single data points, show a simple bar chart
+                single_categories = []
+                single_latencies = []
+                
+                for category, cat_stats in category_results.items():
+                    if cat_stats['query_count'] > 0:
+                        single_categories.append(category.replace('_', ' ').title())
+                        single_latencies.append(cat_stats['average_latency'])
+                
+                if single_categories:
+                    ax4.bar(single_categories, single_latencies, alpha=0.7, 
+                           color=['#1f77b4', '#ff7f0e', '#d62728'][:len(single_categories)])
+                    ax4.set_title('Category Latency (Single Query Each)', fontweight='bold')
+                    ax4.set_ylabel('Latency (seconds)')
+                    ax4.grid(True, alpha=0.3)
+                    ax4.axhline(y=30.0, color='red', linestyle='--', alpha=0.7, label='30s Target')
+                    ax4.legend()
+                else:
+                    ax4.text(0.5, 0.5, 'No data available for distribution plot', 
+                            ha='center', va='center', transform=ax4.transAxes)
+                    ax4.set_title('Latency Distribution', fontweight='bold')
+            
+            # Adjust layout and save
+            plt.tight_layout()
+            
+            # Save chart
+            timestamp = datetime.now().strftime("%Y%m%d_%H%M%S")
+            chart_filename = f"latency_analysis_charts_{timestamp}.png"
+            
+            # Ensure results directory exists
+            results_dir = Path(__file__).parent / "results"
+            results_dir.mkdir(exist_ok=True)
+            chart_path = results_dir / chart_filename
+            
+            plt.savefig(chart_path, dpi=300, bbox_inches='tight', 
+                       facecolor='white', edgecolor='none')
+            plt.close()
+            
+            print(f"📈 Charts saved to: {chart_path}")
+            return str(chart_path)
+            
+        except Exception as e:
+            print(f"❌ Chart generation failed: {e}")
+            return ""
+    
+    def print_statistics_summary(self, stats: Dict[str, Any]):
+        """Print formatted statistics summary to console"""
+        category_results = stats['category_results']
+        overall_results = stats['overall_results']
+        
+        print(f"\n📊 === LATENCY ANALYSIS CHART SUMMARY ===")
+        print(f"Overall Performance:")
+        print(f"   Average Latency: {overall_results['average_latency']:.2f}s (±{overall_results['std_deviation']:.2f})")
+        print(f"   Success Rate: {overall_results['successful_queries']}/{overall_results['total_queries']}")
+        print(f"   30s Target Compliance: {overall_results['target_compliance']:.1%}")
+        
+        print(f"\nCategory Breakdown:")
+        for category, cat_stats in category_results.items():
+            if cat_stats['query_count'] > 0:
+                print(f"   {category.capitalize()}: {cat_stats['average_latency']:.2f}s (±{cat_stats['std_deviation']:.2f}) [{cat_stats['query_count']} queries]")
+
+
+# Independent execution interface
+if __name__ == "__main__":
+    """Independent chart generation interface"""
+    
+    print("📈 OnCall.ai Latency Chart Generator")
+    
+    # Initialize chart generator
+    chart_gen = LatencyChartGenerator()
+    
+    try:
+        # Load latest statistics
+        stats = chart_gen.load_latest_statistics()
+        
+        # Generate charts
+        chart_path = chart_gen.generate_comprehensive_charts(stats)
+        
+        # Print summary
+        chart_gen.print_statistics_summary(stats)
+        
+        print(f"\n✅ Chart generation complete!")
+        print(f"📈 Charts saved to: {chart_path}")
+        
+    except FileNotFoundError as e:
+        print(f"❌ {e}")
+        print("💡 Please run latency_evaluator.py first to generate statistics data")
+    except Exception as e:
+        print(f"❌ Chart generation failed: {e}")

evaluation/latency_evaluator.py

@@ -0,0 +1,496 @@
+#!/usr/bin/env python3
+"""
+OnCall.ai System - Latency Evaluator (Single Query Test Mode)
+============================================================
+
+Test latency for individual queries to avoid rate limits.
+Based on existing system flow: app.py -> user_prompt.py -> retrieval.py -> generation.py
+
+Author: YanBo Chen  
+Date: 2025-08-04
+"""
+
+import time
+import json
+import os
+import sys
+from typing import Dict, List, Any
+from datetime import datetime
+from pathlib import Path
+import re
+
+# Add project path
+current_dir = Path(__file__).parent
+project_root = current_dir.parent
+src_dir = project_root / "src"
+sys.path.insert(0, str(src_dir))
+
+# Import existing system components
+try:
+    from user_prompt import UserPromptProcessor
+    from retrieval import BasicRetrievalSystem
+    from llm_clients import llm_Med42_70BClient
+    from generation import MedicalAdviceGenerator
+except ImportError as e:
+    print(f"❌ Import failed: {e}")
+    print("Please ensure running from project root directory")
+    sys.exit(1)
+
+
+class LatencyEvaluator:
+    """Pure latency measurement and medical advice output recording - no visualization"""
+    
+    def __init__(self):
+        """Initialize existing system components"""
+        print("🔧 Initializing Latency Evaluator...")
+        
+        # Initialize existing system components (same as app.py)
+        self.llm_client = llm_Med42_70BClient()
+        self.retrieval_system = BasicRetrievalSystem()
+        self.user_prompt_processor = UserPromptProcessor(
+            llm_client=self.llm_client,
+            retrieval_system=self.retrieval_system
+        )
+        self.medical_generator = MedicalAdviceGenerator(llm_client=self.llm_client)
+        
+        # Results accumulation for summary statistics
+        self.accumulated_results = {
+            "diagnosis": [],
+            "treatment": [],
+            "mixed": []
+        }
+        
+        # Medical advice outputs for model comparison
+        self.medical_outputs = []
+        
+        print("✅ Latency Evaluator initialization complete")
+    
+    def measure_single_query_latency(self, query: str, category: str = "unknown") -> Dict[str, Any]:
+        """
+        Measure complete processing time for a single query
+        
+        Replicates app.py's process_medical_query flow with timing focus
+        
+        Args:
+            query: Medical query to test
+            category: Query category (diagnosis/treatment/mixed)
+        """
+        print(f"⏱️ Measuring query latency: {query[:50]}...")
+        print(f"📋 Category: {category}")
+        
+        overall_start = time.time()
+        timing_details = {}
+        
+        try:
+            # STEP 1: Condition extraction (user_prompt.py)
+            step1_start = time.time()
+            condition_result = self.user_prompt_processor.extract_condition_keywords(query)
+            timing_details['step1_condition_extraction'] = time.time() - step1_start
+            
+            print(f"   Step 1 - Condition extraction: {timing_details['step1_condition_extraction']:.3f}s")
+            print(f"   Extracted condition: {condition_result.get('condition', 'None')}")
+            
+            # Check if valid medical query
+            if condition_result.get('query_status') in ['invalid_query', 'non_medical']:
+                total_time = time.time() - overall_start
+                print(f"   ⚠️ Non-medical query detected")
+                return {
+                    "query": query,
+                    "category": category,
+                    "total_latency": total_time,
+                    "timing_details": timing_details,
+                    "status": "non_medical",
+                    "condition_result": condition_result,
+                    "success": False,
+                    "timestamp": datetime.now().isoformat()
+                }
+            
+            # STEP 2: User confirmation (simulate auto-confirmation)
+            step2_start = time.time()
+            confirmation = self.user_prompt_processor.handle_user_confirmation(condition_result)
+            timing_details['step2_confirmation'] = time.time() - step2_start
+            
+            print(f"   Step 2 - User confirmation: {timing_details['step2_confirmation']:.3f}s")
+            
+            # STEP 3: Retrieve relevant guidelines (retrieval.py)
+            step3_start = time.time()
+            
+            search_query = f"{condition_result.get('emergency_keywords', '')} {condition_result.get('treatment_keywords', '')}".strip()
+            if not search_query:
+                search_query = condition_result.get('condition', query)
+            
+            retrieval_results = self.retrieval_system.search(search_query, top_k=5)
+            timing_details['step3_retrieval'] = time.time() - step3_start
+            
+            retrieved_count = len(retrieval_results.get('processed_results', []))
+            print(f"   Step 3 - Retrieval: {timing_details['step3_retrieval']:.3f}s ({retrieved_count} results)")
+            
+            # STEP 4: Generate medical advice (generation.py)
+            step4_start = time.time()
+            
+            intention = self._detect_query_intention(query)
+            medical_advice_result = self.medical_generator.generate_medical_advice(
+                user_query=query,
+                retrieval_results=retrieval_results,
+                intention=intention
+            )
+            timing_details['step4_generation'] = time.time() - step4_start
+            
+            print(f"   Step 4 - Generation: {timing_details['step4_generation']:.3f}s")
+            
+            total_time = time.time() - overall_start
+            
+            # Extract medical advice output for future model comparison
+            medical_advice_text = medical_advice_result.get('medical_advice', '')
+            confidence_score = medical_advice_result.get('confidence_score', 0.0)
+            
+            result = {
+                "query": query,
+                "category": category,
+                "total_latency": total_time,
+                "timing_details": timing_details,
+                "condition_result": condition_result,
+                "retrieval_results": retrieval_results,
+                "medical_advice_result": medical_advice_result,
+                "status": "success",
+                "success": True,
+                "timestamp": datetime.now().isoformat()
+            }
+            
+            # Store medical output separately for model comparison
+            medical_output = {
+                "query": query,
+                "category": category,
+                "medical_advice": medical_advice_text,
+                "confidence_score": confidence_score,
+                "query_id": f"{category}_query",
+                "processing_time": total_time,
+                "timestamp": datetime.now().isoformat()
+            }
+            
+            self.medical_outputs.append(medical_output)
+            
+            print(f"✅ Query completed successfully in {total_time:.2f}s")
+            print(f"📝 Medical advice recorded ({len(medical_advice_text)} characters)")
+            
+            return result
+            
+        except Exception as e:
+            total_time = time.time() - overall_start
+            print(f"❌ Query failed after {total_time:.2f}s: {e}")
+            
+            return {
+                "query": query,
+                "category": category,
+                "total_latency": total_time,
+                "timing_details": timing_details,
+                "error": str(e),
+                "status": "error",
+                "success": False,
+                "timestamp": datetime.now().isoformat()
+            }
+    
+    def test_individual_queries_from_file(self, filepath: str) -> Dict[str, List[Dict]]:
+        """
+        Parse queries from file and return them for individual testing
+        
+        Returns categorized queries for separate testing
+        """
+        print(f"📁 Reading queries from file: {filepath}")
+        
+        try:
+            with open(filepath, 'r', encoding='utf-8') as f:
+                content = f.read()
+            
+            # Parse queries with category labels
+            queries_by_category = {
+                "diagnosis": [],
+                "treatment": [], 
+                "mixed": []
+            }
+            
+            lines = content.strip().split('\n')
+            
+            for line in lines:
+                line = line.strip()
+                if not line:
+                    continue
+                
+                # Parse format: "1.diagnosis: query text"
+                match = re.match(r'^\d+\.(diagnosis|treatment|mixed/complicated|mixed):\s*(.+)', line, re.IGNORECASE)
+                if match:
+                    category_raw = match.group(1).lower()
+                    query_text = match.group(2).strip()
+                    
+                    # Normalize category name
+                    if category_raw in ['mixed/complicated', 'mixed']:
+                        category = 'mixed'
+                    else:
+                        category = category_raw
+                    
+                    if category in queries_by_category and len(query_text) > 15:
+                        queries_by_category[category].append({
+                            "text": query_text,
+                            "category": category
+                        })
+            
+            print(f"📋 Parsed queries by category:")
+            for category, category_queries in queries_by_category.items():
+                print(f"  {category.capitalize()}: {len(category_queries)} queries")
+                for i, query_info in enumerate(category_queries):
+                    print(f"    {i+1}. {query_info['text'][:60]}...")
+            
+            return queries_by_category
+            
+        except Exception as e:
+            print(f"❌ Failed to read file: {e}")
+            return {"error": f"Failed to read file: {e}"}
+    
+    def _detect_query_intention(self, query: str) -> str:
+        """Simplified query intention detection (from app.py)"""
+        query_lower = query.lower()
+        
+        if any(word in query_lower for word in ['diagnos', 'differential', 'possible', 'causes']):
+            return 'diagnosis'
+        elif any(word in query_lower for word in ['treat', 'manage', 'therapy', 'intervention']):
+            return 'treatment'
+        else:
+            return 'mixed'
+    
+    def save_single_result(self, result: Dict[str, Any], filename: str = None) -> str:
+        """Save single query evaluation result"""
+        if filename is None:
+            timestamp = datetime.now().strftime("%Y%m%d_%H%M%S")
+            category = result.get('category', 'unknown')
+            filename = f"latency_{category}_{timestamp}.json"
+        
+        # Ensure results directory exists
+        results_dir = Path(__file__).parent / "results"
+        results_dir.mkdir(exist_ok=True)
+        
+        filepath = results_dir / filename
+        
+        with open(filepath, 'w', encoding='utf-8') as f:
+            json.dump(result, f, indent=2, ensure_ascii=False)
+        
+        print(f"💾 Result saved to: {filepath}")
+        return str(filepath)
+
+
+# Independent execution interface
+if __name__ == "__main__":
+    """Independent test interface for single queries"""
+    
+    print("🚀 OnCall.ai Latency Evaluator - Single Query Test Mode")
+    
+    if len(sys.argv) > 1:
+        query_file = sys.argv[1]
+    else:
+        # Default to evaluation/pre_user_query_evaluate.txt
+        query_file = Path(__file__).parent / "pre_user_query_evaluate.txt"
+    
+    if not os.path.exists(query_file):
+        print(f"❌ Query file not found: {query_file}")
+        print("Usage: python latency_evaluator.py [query_file.txt]")
+        sys.exit(1)
+    
+    # Initialize evaluator
+    evaluator = LatencyEvaluator()
+    
+    # Parse queries from file
+    queries_by_category = evaluator.test_individual_queries_from_file(str(query_file))
+    
+    if "error" in queries_by_category:
+        print(f"❌ Failed to parse queries: {queries_by_category['error']}")
+        sys.exit(1)
+    
+    # Test each category individually
+    print(f"\n🧪 Individual Query Testing Mode with Result Accumulation")
+    print(f"📝 Test each query separately to avoid rate limits")
+    
+    for category, queries in queries_by_category.items():
+        if not queries:
+            continue
+            
+        print(f"\n📂 Testing {category.upper()} queries:")
+        
+        for i, query_info in enumerate(queries):
+            query_text = query_info['text']
+            print(f"\n🔍 Query {i+1}/{len(queries)} in {category} category:")
+            print(f"   Text: {query_text}")
+            
+            # Test single query
+            result = evaluator.measure_single_query_latency(query_text, category)
+            
+            # Add to accumulator for chart generation
+            evaluator.add_result_to_accumulator(result)
+            
+            # Save individual result
+            timestamp = datetime.now().strftime("%Y%m%d_%H%M%S")
+            filename = f"latency_{category}_query{i+1}_{timestamp}.json"
+            saved_path = evaluator.save_single_result(result, filename)
+            
+            # Show summary
+            if result.get('success'):
+                print(f"   ✅ Success: {result['total_latency']:.2f}s total")
+                print(f"      Breakdown: Extract={result['timing_details']['step1_condition_extraction']:.2f}s, "
+                      f"Retrieve={result['timing_details']['step3_retrieval']:.2f}s, "
+                      f"Generate={result['timing_details']['step4_generation']:.2f}s")
+            else:
+                print(f"   ❌ Failed: {result.get('status')} - {result.get('error', 'Unknown error')}")
+            
+            # Pause between queries to avoid rate limits
+            if i < len(queries) - 1:  # Not the last query in category
+                print(f"   ⏳ Pausing 5s before next query...")
+                time.sleep(5)
+        
+        # Longer pause between categories
+        if category != list(queries_by_category.keys())[-1]:  # Not the last category
+            print(f"\n⏳ Pausing 10s before next category...")
+            time.sleep(10)
+    
+    # Generate comprehensive analysis (no charts - pure data)
+    print(f"\n📊 Generating comprehensive statistical summary...")
+    
+    # Calculate category statistics
+    final_stats = evaluator.calculate_category_statistics()
+    
+    # Save statistics for chart generation
+    stats_path = evaluator.save_statistics_summary()
+    
+    # Save medical outputs for model comparison
+    outputs_path = evaluator.save_medical_outputs()
+    
+    # Print final summary
+    print(f"\n📊 === FINAL LATENCY ANALYSIS SUMMARY ===")
+    category_results = final_stats['category_results']
+    overall_results = final_stats['overall_results']
+    
+    print(f"Overall Performance:")
+    print(f"   Average Latency: {overall_results['average_latency']:.2f}s (±{overall_results['std_deviation']:.2f})")
+    print(f"   Success Rate: {overall_results['successful_queries']}/{overall_results['total_queries']}")
+    print(f"   30s Target Compliance: {overall_results['target_compliance']:.1%}")
+    
+    print(f"\nCategory Breakdown:")
+    for category, stats in category_results.items():
+        if stats['query_count'] > 0:
+            print(f"   {category.capitalize()}: {stats['average_latency']:.2f}s (±{stats['std_deviation']:.2f}) [{stats['query_count']} queries]")
+    
+    print(f"\n✅ Data collection complete! Files saved:")
+    print(f"   📊 Statistics: {stats_path}")
+    print(f"   📝 Medical Outputs: {outputs_path}")
+    print(f"   📁 Individual results: {Path(__file__).parent / 'results'}")
+    print(f"\n💡 Next step: Run latency_chart_generator.py to create visualizations")
+    
+    def add_result_to_accumulator(self, result: Dict[str, Any]):
+        """Add successful result to category accumulator"""
+        if result.get('success') and result.get('category') in self.accumulated_results:
+            category = result['category']
+            self.accumulated_results[category].append(result)
+            print(f"📊 Added result to {category} category. Total: {len(self.accumulated_results[category])}")
+    
+    def save_statistics_summary(self, filename: str = None) -> str:
+        """Save statistical summary for chart generation"""
+        stats = self.calculate_category_statistics()
+        
+        if filename is None:
+            timestamp = datetime.now().strftime("%Y%m%d_%H%M%S")
+            filename = f"latency_statistics_{timestamp}.json"
+        
+        # Ensure results directory exists
+        results_dir = Path(__file__).parent / "results"
+        results_dir.mkdir(exist_ok=True)
+        
+        filepath = results_dir / filename
+        
+        with open(filepath, 'w', encoding='utf-8') as f:
+            json.dump(stats, f, indent=2, ensure_ascii=False)
+        
+        print(f"📊 Statistics saved to: {filepath}")
+        return str(filepath)
+    
+    def save_medical_outputs(self, filename: str = None) -> str:
+        """Save medical advice outputs for model comparison"""
+        if filename is None:
+            timestamp = datetime.now().strftime("%Y%m%d_%H%M%S")
+            filename = f"medical_outputs_{timestamp}.json"
+        
+        # Ensure results directory exists
+        results_dir = Path(__file__).parent / "results"
+        results_dir.mkdir(exist_ok=True)
+        
+        filepath = results_dir / filename
+        
+        # Create comprehensive output data
+        output_data = {
+            "evaluation_metadata": {
+                "total_outputs": len(self.medical_outputs),
+                "categories": list(set(output['category'] for output in self.medical_outputs)),
+                "timestamp": datetime.now().isoformat(),
+                "model_type": "Med42-70B_RAG_enhanced"  # For future comparison
+            },
+            "medical_outputs": self.medical_outputs
+        }
+        
+        with open(filepath, 'w', encoding='utf-8') as f:
+            json.dump(output_data, f, indent=2, ensure_ascii=False)
+        
+        print(f"📝 Medical outputs saved to: {filepath}")
+        print(f"    Total outputs: {len(self.medical_outputs)}")
+        print(f"    Categories: {', '.join(set(output['category'] for output in self.medical_outputs))}")
+        
+        return str(filepath)
+    
+    def calculate_category_statistics(self) -> Dict[str, Any]:
+        """Calculate statistics for each category and overall"""
+        category_stats = {}
+        all_successful_latencies = []
+        
+        for category, results in self.accumulated_results.items():
+            latencies = [r['total_latency'] for r in results if r.get('success')]
+            
+            if latencies:
+                category_stats[category] = {
+                    "average_latency": sum(latencies) / len(latencies),
+                    "std_deviation": self._calculate_std(latencies),
+                    "min_latency": min(latencies),
+                    "max_latency": max(latencies),
+                    "query_count": len(latencies),
+                    "individual_latencies": latencies
+                }
+                all_successful_latencies.extend(latencies)
+            else:
+                category_stats[category] = {
+                    "average_latency": 0.0,
+                    "std_deviation": 0.0,
+                    "min_latency": 0.0,
+                    "max_latency": 0.0,
+                    "query_count": 0,
+                    "individual_latencies": []
+                }
+        
+        # Calculate overall statistics
+        overall_stats = {
+            "average_latency": sum(all_successful_latencies) / len(all_successful_latencies) if all_successful_latencies else 0.0,
+            "std_deviation": self._calculate_std(all_successful_latencies),
+            "min_latency": min(all_successful_latencies) if all_successful_latencies else 0.0,
+            "max_latency": max(all_successful_latencies) if all_successful_latencies else 0.0,
+            "total_queries": sum(len(results) for results in self.accumulated_results.values()),
+            "successful_queries": len(all_successful_latencies),
+            "target_compliance": sum(1 for lat in all_successful_latencies if lat <= 30.0) / len(all_successful_latencies) if all_successful_latencies else 0.0
+        }
+        
+        return {
+            "category_results": category_stats,
+            "overall_results": overall_stats,
+            "timestamp": datetime.now().isoformat()
+        }
+    
+    def _calculate_std(self, values: List[float]) -> float:
+        """Calculate standard deviation"""
+        if len(values) < 2:
+            return 0.0
+        
+        mean = sum(values) / len(values)
+        variance = sum((x - mean) ** 2 for x in values) / len(values)
+        return variance ** 0.5

evaluation/pre_user_query_evaluate.txt

@@ -0,0 +1,5 @@
+1.diagnosis: 60-year-old patient with hypertension history, sudden chest pain. What are possible causes and how to assess?
+
+2.treatment: Suspected acute ischemic stroke. Tell me the next steps to take
+
+3.mixed/complicated: 20 y/f , porphyria, sudden seizure. What are possible causes and complete management workflow?

evaluation/relevance_evaluator.py

@@ -0,0 +1,447 @@
+#!/usr/bin/env python3
+"""
+OnCall.ai System - Retrieval Relevance Evaluator (Metric 3)
+===========================================================
+
+Evaluates retrieval relevance using cosine similarity from retrieval.py
+Automatic evaluation based on existing similarity scores with optional LLM sampling
+
+Author: YanBo Chen  
+Date: 2025-08-04
+"""
+
+import json
+import os
+import sys
+from typing import Dict, List, Any
+from datetime import datetime
+from pathlib import Path
+import re
+import numpy as np
+
+# Add project path
+current_dir = Path(__file__).parent
+project_root = current_dir.parent
+src_dir = project_root / "src"
+sys.path.insert(0, str(src_dir))
+
+# Import existing system components
+try:
+    from user_prompt import UserPromptProcessor
+    from retrieval import BasicRetrievalSystem
+    from llm_clients import llm_Med42_70BClient
+except ImportError as e:
+    print(f"❌ Import failed: {e}")
+    print("Please ensure running from project root directory")
+    sys.exit(1)
+
+
+class RelevanceEvaluator:
+    """Retrieval relevance evaluator using cosine similarity - automatic evaluation"""
+    
+    def __init__(self):
+        """Initialize system components for relevance testing"""
+        print("🔧 Initializing Relevance Evaluator...")
+        
+        # Initialize required components
+        self.llm_client = llm_Med42_70BClient()
+        self.retrieval_system = BasicRetrievalSystem()
+        self.user_prompt_processor = UserPromptProcessor(
+            llm_client=self.llm_client,
+            retrieval_system=self.retrieval_system
+        )
+        
+        # Results accumulation
+        self.relevance_results = []
+        
+        print("✅ Relevance Evaluator initialization complete")
+    
+    def evaluate_single_relevance(self, query: str, category: str = "unknown") -> Dict[str, Any]:
+        """
+        Evaluate retrieval relevance for a single query
+        
+        Uses existing cosine similarity scores from retrieval.py
+        
+        Args:
+            query: Medical query to test
+            category: Query category (diagnosis/treatment/mixed)
+        """
+        print(f"🔍 Testing relevance for: {query[:50]}...")
+        print(f"📋 Category: {category}")
+        
+        try:
+            # Step 1: Extract condition for search query construction
+            condition_result = self.user_prompt_processor.extract_condition_keywords(query)
+            
+            # Step 2: Perform retrieval (same as latency_evaluator.py)
+            search_query = f"{condition_result.get('emergency_keywords', '')} {condition_result.get('treatment_keywords', '')}".strip()
+            if not search_query:
+                search_query = condition_result.get('condition', query)
+            
+            retrieval_start = datetime.now()
+            retrieval_results = self.retrieval_system.search(search_query, top_k=5)
+            retrieval_time = (datetime.now() - retrieval_start).total_seconds()
+            
+            # Step 3: Extract similarity scores from retrieval results
+            processed_results = retrieval_results.get('processed_results', [])
+            
+            if not processed_results:
+                result = {
+                    "query": query,
+                    "category": category,
+                    "search_query": search_query,
+                    "retrieval_success": False,
+                    "average_relevance": 0.0,
+                    "relevance_scores": [],
+                    "retrieved_count": 0,
+                    "retrieval_time": retrieval_time,
+                    "error": "No retrieval results",
+                    "timestamp": datetime.now().isoformat()
+                }
+                
+                self.relevance_results.append(result)
+                print(f"   ❌ No retrieval results found")
+                return result
+            
+            # Extract cosine similarity scores
+            similarity_scores = []
+            retrieval_details = []
+            
+            for i, doc_result in enumerate(processed_results):
+                # Get similarity score (may be stored as 'distance', 'similarity_score', or 'score')
+                similarity = (
+                    doc_result.get('distance', 0.0) or 
+                    doc_result.get('similarity_score', 0.0) or
+                    doc_result.get('score', 0.0)
+                )
+                
+                similarity_scores.append(similarity)
+                
+                retrieval_details.append({
+                    "doc_index": i,
+                    "similarity_score": similarity,
+                    "content_snippet": doc_result.get('content', '')[:100] + "...",
+                    "doc_type": doc_result.get('type', 'unknown'),
+                    "source": doc_result.get('source', 'unknown')
+                })
+            
+            # Calculate relevance metrics
+            average_relevance = sum(similarity_scores) / len(similarity_scores)
+            max_relevance = max(similarity_scores)
+            min_relevance = min(similarity_scores)
+            
+            # Count high-relevance results (threshold: 0.2 based on evaluation_instruction.md)
+            high_relevance_count = sum(1 for score in similarity_scores if score >= 0.2)
+            high_relevance_ratio = high_relevance_count / len(similarity_scores)
+            
+            result = {
+                "query": query,
+                "category": category,
+                "search_query": search_query,
+                "retrieval_success": True,
+                "average_relevance": average_relevance,
+                "max_relevance": max_relevance,
+                "min_relevance": min_relevance,
+                "relevance_scores": similarity_scores,
+                "high_relevance_count": high_relevance_count,
+                "high_relevance_ratio": high_relevance_ratio,
+                "retrieved_count": len(processed_results),
+                "retrieval_time": retrieval_time,
+                "retrieval_details": retrieval_details,
+                "meets_threshold": average_relevance >= 0.2,
+                "timestamp": datetime.now().isoformat()
+            }
+            
+            # Store result
+            self.relevance_results.append(result)
+            
+            print(f"   ✅ Retrieval: {len(processed_results)} documents")
+            print(f"   📊 Average Relevance: {average_relevance:.3f}")
+            print(f"   📈 High Relevance (≥0.2): {high_relevance_count}/{len(processed_results)} ({high_relevance_ratio:.1%})")
+            print(f"   🎯 Threshold: {'✅ Met' if result['meets_threshold'] else '❌ Not Met'}")
+            print(f"   ⏱️ Retrieval Time: {retrieval_time:.3f}s")
+            
+            return result
+            
+        except Exception as e:
+            error_result = {
+                "query": query,
+                "category": category,
+                "retrieval_success": False,
+                "average_relevance": 0.0,
+                "error": str(e),
+                "timestamp": datetime.now().isoformat()
+            }
+            
+            self.relevance_results.append(error_result)
+            print(f"   ❌ Relevance evaluation failed: {e}")
+            
+            return error_result
+    
+    def parse_queries_from_file(self, filepath: str) -> Dict[str, List[Dict]]:
+        """Parse queries from file with category labels"""
+        print(f"📁 Reading queries from file: {filepath}")
+        
+        try:
+            with open(filepath, 'r', encoding='utf-8') as f:
+                content = f.read()
+            
+            # Parse queries with category labels
+            queries_by_category = {
+                "diagnosis": [],
+                "treatment": [], 
+                "mixed": []
+            }
+            
+            lines = content.strip().split('\n')
+            
+            for line in lines:
+                line = line.strip()
+                if not line:
+                    continue
+                
+                # Parse format: "1.diagnosis: query text"
+                match = re.match(r'^\d+\.(diagnosis|treatment|mixed/complicated|mixed):\s*(.+)', line, re.IGNORECASE)
+                if match:
+                    category_raw = match.group(1).lower()
+                    query_text = match.group(2).strip()
+                    
+                    # Normalize category name
+                    if category_raw in ['mixed/complicated', 'mixed']:
+                        category = 'mixed'
+                    else:
+                        category = category_raw
+                    
+                    if category in queries_by_category and len(query_text) > 15:
+                        queries_by_category[category].append({
+                            "text": query_text,
+                            "category": category
+                        })
+            
+            print(f"📋 Parsed queries by category:")
+            for category, category_queries in queries_by_category.items():
+                print(f"  {category.capitalize()}: {len(category_queries)} queries")
+            
+            return queries_by_category
+            
+        except Exception as e:
+            print(f"❌ Failed to read file: {e}")
+            return {"error": f"Failed to read file: {e}"}
+    
+    def calculate_relevance_statistics(self) -> Dict[str, Any]:
+        """Calculate relevance statistics by category"""
+        category_stats = {}
+        all_successful_results = []
+        
+        # Group results by category
+        results_by_category = {
+            "diagnosis": [],
+            "treatment": [],
+            "mixed": []
+        }
+        
+        for result in self.relevance_results:
+            category = result.get('category', 'unknown')
+            if category in results_by_category:
+                results_by_category[category].append(result)
+                if result.get('retrieval_success'):
+                    all_successful_results.append(result)
+        
+        # Calculate statistics for each category
+        for category, results in results_by_category.items():
+            successful_results = [r for r in results if r.get('retrieval_success')]
+            
+            if successful_results:
+                avg_relevance = sum(r['average_relevance'] for r in successful_results) / len(successful_results)
+                relevance_scores = [r['average_relevance'] for r in successful_results]
+                avg_retrieval_time = sum(r.get('retrieval_time', 0) for r in successful_results) / len(successful_results)
+                
+                category_stats[category] = {
+                    "average_relevance": avg_relevance,
+                    "max_relevance": max(relevance_scores),
+                    "min_relevance": min(relevance_scores),
+                    "successful_retrievals": len(successful_results),
+                    "total_queries": len(results),
+                    "success_rate": len(successful_results) / len(results),
+                    "average_retrieval_time": avg_retrieval_time,
+                    "meets_threshold": avg_relevance >= 0.2,
+                    "individual_relevance_scores": relevance_scores
+                }
+            else:
+                category_stats[category] = {
+                    "average_relevance": 0.0,
+                    "max_relevance": 0.0,
+                    "min_relevance": 0.0,
+                    "successful_retrievals": 0,
+                    "total_queries": len(results),
+                    "success_rate": 0.0,
+                    "average_retrieval_time": 0.0,
+                    "meets_threshold": False,
+                    "individual_relevance_scores": []
+                }
+        
+        # Calculate overall statistics
+        if all_successful_results:
+            all_relevance_scores = [r['average_relevance'] for r in all_successful_results]
+            overall_stats = {
+                "average_relevance": sum(all_relevance_scores) / len(all_relevance_scores),
+                "max_relevance": max(all_relevance_scores),
+                "min_relevance": min(all_relevance_scores),
+                "successful_retrievals": len(all_successful_results),
+                "total_queries": len(self.relevance_results),
+                "success_rate": len(all_successful_results) / len(self.relevance_results),
+                "meets_threshold": (sum(all_relevance_scores) / len(all_relevance_scores)) >= 0.2,
+                "target_compliance": (sum(all_relevance_scores) / len(all_relevance_scores)) >= 0.25
+            }
+        else:
+            overall_stats = {
+                "average_relevance": 0.0,
+                "max_relevance": 0.0,
+                "min_relevance": 0.0,
+                "successful_retrievals": 0,
+                "total_queries": len(self.relevance_results),
+                "success_rate": 0.0,
+                "meets_threshold": False,
+                "target_compliance": False
+            }
+        
+        return {
+            "category_results": category_stats,
+            "overall_results": overall_stats,
+            "timestamp": datetime.now().isoformat()
+        }
+    
+    def save_relevance_statistics(self, filename: str = None) -> str:
+        """Save relevance statistics for chart generation"""
+        stats = self.calculate_relevance_statistics()
+        
+        if filename is None:
+            timestamp = datetime.now().strftime("%Y%m%d_%H%M%S")
+            filename = f"relevance_statistics_{timestamp}.json"
+        
+        # Ensure results directory exists
+        results_dir = Path(__file__).parent / "results"
+        results_dir.mkdir(exist_ok=True)
+        
+        filepath = results_dir / filename
+        
+        with open(filepath, 'w', encoding='utf-8') as f:
+            json.dump(stats, f, indent=2, ensure_ascii=False)
+        
+        print(f"📊 Relevance statistics saved to: {filepath}")
+        return str(filepath)
+    
+    def save_relevance_details(self, filename: str = None) -> str:
+        """Save detailed relevance results"""
+        if filename is None:
+            timestamp = datetime.now().strftime("%Y%m%d_%H%M%S")
+            filename = f"relevance_details_{timestamp}.json"
+        
+        # Ensure results directory exists
+        results_dir = Path(__file__).parent / "results"
+        results_dir.mkdir(exist_ok=True)
+        
+        filepath = results_dir / filename
+        
+        # Create comprehensive relevance data
+        relevance_data = {
+            "evaluation_metadata": {
+                "total_queries": len(self.relevance_results),
+                "successful_retrievals": len([r for r in self.relevance_results if r.get('retrieval_success')]),
+                "timestamp": datetime.now().isoformat(),
+                "evaluator_type": "retrieval_relevance",
+                "threshold_used": 0.2
+            },
+            "relevance_results": self.relevance_results
+        }
+        
+        with open(filepath, 'w', encoding='utf-8') as f:
+            json.dump(relevance_data, f, indent=2, ensure_ascii=False)
+        
+        print(f"📝 Relevance details saved to: {filepath}")
+        return str(filepath)
+
+
+# Independent execution interface
+if __name__ == "__main__":
+    """Independent relevance evaluation interface"""
+    
+    print("📊 OnCall.ai Relevance Evaluator - Retrieval Relevance Analysis")
+    
+    if len(sys.argv) > 1:
+        query_file = sys.argv[1]
+    else:
+        # Default to evaluation/pre_user_query_evaluate.txt
+        query_file = Path(__file__).parent / "pre_user_query_evaluate.txt"
+    
+    if not os.path.exists(query_file):
+        print(f"❌ Query file not found: {query_file}")
+        print("Usage: python relevance_evaluator.py [query_file.txt]")
+        sys.exit(1)
+    
+    # Initialize evaluator
+    evaluator = RelevanceEvaluator()
+    
+    # Parse queries from file
+    queries_by_category = evaluator.parse_queries_from_file(str(query_file))
+    
+    if "error" in queries_by_category:
+        print(f"❌ Failed to parse queries: {queries_by_category['error']}")
+        sys.exit(1)
+    
+    # Test relevance for each query
+    print(f"\n🧪 Retrieval Relevance Testing")
+    
+    for category, queries in queries_by_category.items():
+        if not queries:
+            continue
+            
+        print(f"\n📂 Testing {category.upper()} relevance:")
+        
+        for i, query_info in enumerate(queries):
+            query_text = query_info['text']
+            
+            # Test relevance
+            result = evaluator.evaluate_single_relevance(query_text, category)
+            
+            # Pause between queries to avoid rate limits
+            if i < len(queries) - 1:
+                print(f"   ⏳ Pausing 3s before next query...")
+                import time
+                time.sleep(3)
+        
+        # Pause between categories
+        if category != list(queries_by_category.keys())[-1]:
+            print(f"\n⏳ Pausing 5s before next category...")
+            import time
+            time.sleep(5)
+    
+    # Generate and save results
+    print(f"\n📊 Generating relevance analysis...")
+    
+    # Save statistics and details
+    stats_path = evaluator.save_relevance_statistics()
+    details_path = evaluator.save_relevance_details()
+    
+    # Print final summary
+    stats = evaluator.calculate_relevance_statistics()
+    category_results = stats['category_results']
+    overall_results = stats['overall_results']
+    
+    print(f"\n📊 === RELEVANCE EVALUATION SUMMARY ===")
+    print(f"Overall Performance:")
+    print(f"   Average Relevance: {overall_results['average_relevance']:.3f}")
+    print(f"   Retrieval Success Rate: {overall_results['success_rate']:.1%}")
+    print(f"   0.2 Threshold: {'✅ Met' if overall_results['meets_threshold'] else '❌ Not Met'}")
+    print(f"   0.25 Target: {'✅ Met' if overall_results['target_compliance'] else '❌ Not Met'}")
+    
+    print(f"\nCategory Breakdown:")
+    for category, cat_stats in category_results.items():
+        if cat_stats['total_queries'] > 0:
+            print(f"   {category.capitalize()}: {cat_stats['average_relevance']:.3f} "
+                  f"({cat_stats['successful_retrievals']}/{cat_stats['total_queries']}) "
+                  f"[{cat_stats['average_retrieval_time']:.3f}s avg]")
+    
+    print(f"\n✅ Relevance evaluation complete!")
+    print(f"📊 Statistics: {stats_path}")
+    print(f"📝 Details: {details_path}")