Refactor relevance calculation and update thresholds in latency evaluator; enhance precision and MRR analyzer with angular distance metrics; increase timeout for primary generation in fallback configuration.

evaluation/latency_evaluator.py

@@ -273,27 +273,25 @@ class ComprehensiveEvaluator:
             
             # METRIC 3: Retrieval Relevance Analysis
             if processed_results:
-                similarity_scores = []
+                relevance_scores = []
                 for doc_result in processed_results:
-                    similarity = (
-                        doc_result.get('distance', 0.0) or 
-                        doc_result.get('similarity_score', 0.0) or
-                        doc_result.get('score', 0.0)
-                    )
-                    similarity_scores.append(similarity)
+                    # Get angular distance and convert to relevance using correct formula
+                    distance = doc_result.get('distance', 1.0)
+                    relevance = 1.0 - (distance**2) / 2.0  # Correct mathematical conversion
+                    relevance_scores.append(relevance)
                 
-                average_relevance = sum(similarity_scores) / len(similarity_scores)
-                high_relevance_count = sum(1 for score in similarity_scores if score >= 0.2)
+                average_relevance = sum(relevance_scores) / len(relevance_scores)
+                high_relevance_count = sum(1 for score in relevance_scores if score >= 0.85)
                 
                 relevance_metrics = {
                     "average_relevance": average_relevance,
-                    "max_relevance": max(similarity_scores),
-                    "min_relevance": min(similarity_scores),
-                    "similarity_scores": similarity_scores,
+                    "max_relevance": max(relevance_scores),
+                    "min_relevance": min(relevance_scores),
+                    "relevance_scores": relevance_scores,
                     "high_relevance_count": high_relevance_count,
-                    "high_relevance_ratio": high_relevance_count / len(similarity_scores),
+                    "high_relevance_ratio": high_relevance_count / len(relevance_scores),
                     "retrieved_count": len(processed_results),
-                    "meets_threshold": average_relevance >= 0.2,
+                    "meets_threshold": average_relevance >= 0.85,
                     "retrieval_time": step3_time
                 }
             else:
@@ -322,7 +320,7 @@ class ComprehensiveEvaluator:
                 "latency_metrics": {
                     "total_latency": total_time,
                     "timing_details": timing_details,
-                    "meets_target": total_time <= 30.0
+                    "meets_target": total_time <= 60.0
                 },
                 
                 # Metric 2: Condition Extraction - Success rate from user_prompt.py
@@ -411,7 +409,7 @@ class ComprehensiveEvaluator:
             "latency_metrics": {
                 "total_latency": total_time,
                 "timing_details": timing_details,
-                "meets_target": total_time <= 30.0
+                "meets_target": total_time <= 60.0
             },
             
             # Metric 2: Condition Extraction - Partial data may be available before failure
@@ -546,7 +544,7 @@ class ComprehensiveEvaluator:
                         "min_latency": min(latencies),
                         "max_latency": max(latencies),
                         "query_count": len(latencies),
-                        "target_compliance": sum(1 for lat in latencies if lat <= 30.0) / len(latencies),
+                        "target_compliance": sum(1 for lat in latencies if lat <= 60.0) / len(latencies),
                         "individual_latencies": latencies
                     }
                 else:
@@ -661,7 +659,7 @@ class ComprehensiveEvaluator:
                 "max_latency": max(latencies),
                 "successful_queries": len(all_successful_results),
                 "total_queries": total_queries,
-                "target_compliance": sum(1 for lat in latencies if lat <= 30.0) / len(latencies)
+                "target_compliance": sum(1 for lat in latencies if lat <= 60.0) / len(latencies)
             }
         
         elif metric_name == "extraction":
@@ -682,8 +680,8 @@ class ComprehensiveEvaluator:
                 "min_relevance": min(relevance_scores),
                 "successful_queries": len(all_successful_results),
                 "total_queries": total_queries,
-                "meets_threshold": (sum(relevance_scores) / len(relevance_scores)) >= 0.2,
-                "target_compliance": (sum(relevance_scores) / len(relevance_scores)) >= 0.25
+                "meets_threshold": (sum(relevance_scores) / len(relevance_scores)) >= 0.85,
+                "target_compliance": (sum(relevance_scores) / len(relevance_scores)) >= 0.7
             }
         
         elif metric_name == "coverage" and all_successful_results:
@@ -866,7 +864,7 @@ if __name__ == "__main__":
         
         if metric_name == "latency":
             print(f"   Average: {overall_results['average_latency']:.2f}s (±{overall_results['std_deviation']:.2f})")
-            print(f"   30s Target: {'✅ Met' if overall_results['target_compliance'] >= 0.8 else '❌ Not Met'}")
+            print(f"   60s Target: {'✅ Met' if overall_results['target_compliance'] >= 0.8 else '❌ Not Met'}")
         
         elif metric_name == "extraction":
             print(f"   Success Rate: {overall_results['success_rate']:.1%}")

evaluation/metric7_8_precision_MRR.py

@@ -6,6 +6,12 @@ OnCall.ai System - Precision & MRR Analyzer (Metrics 7-8)
 Specialized analyzer for calculating Precision@K and Mean Reciprocal Rank (MRR)
 using data collected from latency_evaluator.py comprehensive evaluation.
 
+IMPORTANT CHANGES - Angular Distance & Relevance Calculation:
+- DISTANCE METRIC: Uses Angular Distance from Annoy index (range: 0.0-1.0, smaller = more relevant)
+- RELEVANCE CONVERSION: relevance = 1.0 - (angular_distance²) / 2.0 (mathematical correct formula)
+- THRESHOLD ALIGNMENT: Aligned with Metric 3 relevance calculation standards
+- DISPLAY UPDATE: Changed from "Relevance: X" to "Angular Distance: X" for clarity
+
 METRICS CALCULATED:
 7. Precision@K (檢索精確率) - Proportion of relevant results in top-K retrieval
 8. Mean Reciprocal Rank (平均倒數排名) - Average reciprocal rank of first relevant result
@@ -18,6 +24,7 @@ DESIGN PRINCIPLE:
 
 Author: YanBo Chen  
 Date: 2025-08-04
+Updated: 2025-08-04 (Angular Distance alignment)
 """
 
 import json
@@ -121,8 +128,8 @@ class PrecisionMRRAnalyzer:
         # Step 1: Determine query complexity
         is_complex = self._is_complex_query(query, processed_results)
         
-        # Step 2: Choose adaptive threshold
-        threshold = 0.15 if is_complex else 0.25
+        # Step 2: Choose adaptive threshold (aligned with Metric 3 relevance calculation)
+        threshold = 0.75 if is_complex else 0.8
         
         print(f"   🎯 Using relevance threshold: {threshold} ({'lenient' if is_complex else 'strict'})")
         

src/generation.py

@@ -30,7 +30,7 @@ logger = logging.getLogger(__name__)
 
 # Fallback Generation Configuration (Simplified Architecture)
 FALLBACK_TIMEOUTS = {
-    "primary": 30.0,        # Primary Med42-70B with full RAG context
+    "primary": 60.0,        # Primary Med42-70B increased timeout for stable evaluation
     "fallback_1": 1.0,      # RAG template generation (renamed from fallback_2)
     "fallback_2": 0.1       # Minimal template generation (instant)
 }
@@ -279,7 +279,7 @@ class MedicalAdviceGenerator:
             
             # Format each chunk with metadata
             context_part = f"""
-            [Guideline {i}] (Source: {chunk_type.title()}, Relevance: {1-distance:.3f})
+            [Guideline {i}] (Source: {chunk_type.title()}, Angular Distance: {distance:.3f})
             {chunk_text}
             """.strip()