Update logic and add tests

.github/scripts/test_wer_regression_check.py

@@ -0,0 +1,483 @@
+#!/usr/bin/env python3
+"""
+Test script for WER regression detection
+Tests all regression detection functions with synthetic and real data
+"""
+
+import json
+import sys
+from wer_regression_check import (
+    detect_device_regressions,
+    detect_os_regressions,
+    detect_release_regressions,
+    detect_speed_device_regressions,
+    detect_speed_os_regressions,
+    detect_speed_release_regressions,
+    detect_tokens_device_regressions,
+    detect_tokens_os_regressions,
+    detect_tokens_release_regressions,
+    generate_slack_message,
+    load_performance_data
+)
+
+
+def test_wer_detection_with_synthetic_data():
+    """Test WER detection with known synthetic data"""
+    print("\n" + "="*80)
+    print("TEST 1: WER Detection with Synthetic Data")
+    print("="*80)
+    
+    # Create synthetic data where we know there should be regressions
+    # Historical data (best performances)
+    historical_data = [
+        # Model A: iPhone has best WER of 10%
+        {"model": "model-a", "device": "iPhone 15", "os": "iOS 18", "average_wer": 10.0, "speed": 10.0, "tokens_per_second": 50.0},
+        {"model": "model-a", "device": "iPhone 15", "os": "iOS 18", "average_wer": 10.5, "speed": 9.5, "tokens_per_second": 48.0},
+        {"model": "model-a", "device": "iPad Pro", "os": "iOS 18", "average_wer": 10.2, "speed": 9.8, "tokens_per_second": 49.0},
+        
+        # Model B: iOS 17 has best WER of 10%
+        {"model": "model-b", "device": "iPhone 15", "os": "iOS 17", "average_wer": 10.0, "speed": 20.0, "tokens_per_second": 100.0},
+        {"model": "model-b", "device": "iPhone 15", "os": "iOS 18", "average_wer": 10.5, "speed": 19.0, "tokens_per_second": 95.0},
+        
+        # Model C: No regression scenario
+        {"model": "model-c", "device": "iPhone 15", "os": "iOS 18", "average_wer": 10.0, "speed": 10.0, "tokens_per_second": 50.0},
+        {"model": "model-c", "device": "iPad Pro", "os": "iOS 18", "average_wer": 10.5, "speed": 9.5, "tokens_per_second": 48.0},
+    ]
+    
+    # Current data (latest release with regressions)
+    current_data = [
+        # Model A: iPad Pro has regressed to 15% WER (50% worse than best 10%)
+        {"model": "model-a", "device": "iPad Pro", "os": "iOS 18", "average_wer": 15.0, "speed": 8.0, "tokens_per_second": 40.0},
+        {"model": "model-a", "device": "iPhone 15", "os": "iOS 18", "average_wer": 10.3, "speed": 9.7, "tokens_per_second": 49.5},
+        
+        # Model B: iOS 18 has regressed to 13% WER (30% worse than best 10%)
+        {"model": "model-b", "device": "iPhone 15", "os": "iOS 18", "average_wer": 13.0, "speed": 15.0, "tokens_per_second": 75.0},
+        
+        # Model C: Still within 20% (11% vs best 10%)
+        {"model": "model-c", "device": "iPad Pro", "os": "iOS 18", "average_wer": 11.0, "speed": 9.0, "tokens_per_second": 45.0},
+    ]
+    
+    # Test device regressions
+    device_regressions = detect_device_regressions(current_data, historical_data, threshold=20.0)
+    print(f"\n✓ Device WER Regressions Found: {len(device_regressions)}")
+    
+    # Debug: print all found regressions
+    for r in device_regressions:
+        print(f"  - {r['model']}: {r['device']} has {r['current_value']}% WER vs best {r['best_value']}% (diff: {r['percentage_diff']}%)")
+    
+    # Model A should trigger (iPad Pro is ~40% worse than iPhone)
+    # Model C should NOT trigger (iPad Pro is only 10% worse)
+    assert len(device_regressions) >= 1, f"Expected at least 1 device regression, got {len(device_regressions)}"
+    
+    # Verify model-a is in the regressions
+    model_a_regressions = [r for r in device_regressions if r["model"] == "model-a"]
+    assert len(model_a_regressions) > 0, "Expected model-a to have device regression"
+    print(f"\n✓ Model-a correctly flagged for device regression")
+    
+    # Test OS regressions
+    os_regressions = detect_os_regressions(current_data, historical_data, threshold=20.0)
+    print(f"\n✓ OS WER Regressions Found: {len(os_regressions)}")
+    
+    # Debug: print all found OS regressions
+    for r in os_regressions:
+        print(f"  - {r['model']}: {r['os']} has {r['current_value']}% WER vs best {r['best_value']}% (diff: {r['percentage_diff']}%)")
+    
+    assert len(os_regressions) >= 1, f"Expected at least 1 OS regression, got {len(os_regressions)}"
+    
+    # Verify model-b is in the regressions
+    model_b_regressions = [r for r in os_regressions if r["model"] == "model-b"]
+    assert len(model_b_regressions) > 0, "Expected model-b to have OS regression"
+    print(f"\n✓ Model-b correctly flagged for OS regression")
+    
+    print("\n✅ TEST 1 PASSED: WER detection works correctly with synthetic data")
+    return True
+
+
+def test_speed_detection_with_synthetic_data():
+    """Test speed detection with known synthetic data"""
+    print("\n" + "="*80)
+    print("TEST 2: Speed Detection with Synthetic Data")
+    print("="*80)
+    
+    # Historical data (best performances)
+    historical_data = [
+        # Model A: iPhone has best speed of 100
+        {"model": "model-a", "device": "iPhone 15", "os": "iOS 18", "average_wer": 10.0, "speed": 100.0, "tokens_per_second": 200.0},
+        {"model": "model-a", "device": "iPhone 15", "os": "iOS 18", "average_wer": 10.5, "speed": 95.0, "tokens_per_second": 190.0},
+        {"model": "model-a", "device": "iPad Pro", "os": "iOS 18", "average_wer": 11.0, "speed": 98.0, "tokens_per_second": 195.0},
+    ]
+    
+    # Current data (with speed regression)
+    current_data = [
+        # Model A: iPad Pro has regressed to 60 speed (40% slower than best 100)
+        {"model": "model-a", "device": "iPad Pro", "os": "iOS 18", "average_wer": 11.0, "speed": 60.0, "tokens_per_second": 120.0},
+        {"model": "model-a", "device": "iPhone 15", "os": "iOS 18", "average_wer": 10.2, "speed": 97.0, "tokens_per_second": 195.0},
+    ]
+    
+    # Test device speed regressions
+    speed_device_regressions = detect_speed_device_regressions(current_data, historical_data, threshold=20.0)
+    print(f"\n✓ Device Speed Regressions Found: {len(speed_device_regressions)}")
+    assert len(speed_device_regressions) == 1, f"Expected 1 speed device regression, got {len(speed_device_regressions)}"
+    print(f"  - {speed_device_regressions[0]['model']}: {speed_device_regressions[0]['device']} has {speed_device_regressions[0]['current_value']}x speed vs best {speed_device_regressions[0]['best_value']}x")
+    
+    print("\n✅ TEST 2 PASSED: Speed detection works correctly with synthetic data")
+    return True
+
+
+def test_tokens_detection_with_synthetic_data():
+    """Test tokens per second detection with known synthetic data"""
+    print("\n" + "="*80)
+    print("TEST 3: Tokens/Second Detection with Synthetic Data")
+    print("="*80)
+    
+    # Historical data (best performances)
+    historical_data = [
+        # Model A: iPhone has best tokens/sec of 500
+        {"model": "model-a", "device": "iPhone 15", "os": "iOS 18", "average_wer": 10.0, "speed": 100.0, "tokens_per_second": 500.0},
+        {"model": "model-a", "device": "iPad Pro", "os": "iOS 18", "average_wer": 10.0, "speed": 98.0, "tokens_per_second": 490.0},
+    ]
+    
+    # Current data (with tokens/sec regression)
+    current_data = [
+        # Model A: iPad Pro has regressed to 300 tokens/sec (40% slower than best 500)
+        {"model": "model-a", "device": "iPad Pro", "os": "iOS 18", "average_wer": 10.0, "speed": 80.0, "tokens_per_second": 300.0},
+        {"model": "model-a", "device": "iPhone 15", "os": "iOS 18", "average_wer": 10.1, "speed": 99.0, "tokens_per_second": 495.0},
+    ]
+    
+    # Test device tokens regressions
+    tokens_device_regressions = detect_tokens_device_regressions(current_data, historical_data, threshold=20.0)
+    print(f"\n✓ Device Tokens/Sec Regressions Found: {len(tokens_device_regressions)}")
+    assert len(tokens_device_regressions) == 1, f"Expected 1 tokens device regression, got {len(tokens_device_regressions)}"
+    print(f"  - {tokens_device_regressions[0]['model']}: {tokens_device_regressions[0]['device']} has {tokens_device_regressions[0]['current_value']} tokens/sec vs best {tokens_device_regressions[0]['best_value']}")
+    
+    print("\n✅ TEST 3 PASSED: Tokens/sec detection works correctly with synthetic data")
+    return True
+
+
+def test_release_regression_detection():
+    """Test release-to-release regression detection"""
+    print("\n" + "="*80)
+    print("TEST 4: Release-to-Release Regression Detection")
+    print("="*80)
+    
+    # Previous release data (best performance)
+    previous_data = [
+        {"model": "model-a", "device": "iPhone 15", "os": "iOS 18", "average_wer": 10.0, "speed": 100.0, "tokens_per_second": 500.0},
+        {"model": "model-a", "device": "iPhone 15", "os": "iOS 18", "average_wer": 10.5, "speed": 95.0, "tokens_per_second": 490.0},
+    ]
+    
+    # Current release data (degraded performance - 50% worse)
+    current_data = [
+        {"model": "model-a", "device": "iPhone 15", "os": "iOS 18", "average_wer": 15.0, "speed": 60.0, "tokens_per_second": 300.0},
+    ]
+    
+    # Test WER release regression
+    wer_release_regressions = detect_release_regressions(current_data, previous_data, threshold=20.0)
+    print(f"\n✓ WER Release Regressions Found: {len(wer_release_regressions)}")
+    assert len(wer_release_regressions) == 1, f"Expected 1 WER release regression, got {len(wer_release_regressions)}"
+    print(f"  - {wer_release_regressions[0]['model']}: WER increased from {wer_release_regressions[0]['best_historical_value']}% to {wer_release_regressions[0]['current_value']}%")
+    
+    # Test speed release regression
+    speed_release_regressions = detect_speed_release_regressions(current_data, previous_data, threshold=20.0)
+    print(f"\n✓ Speed Release Regressions Found: {len(speed_release_regressions)}")
+    assert len(speed_release_regressions) == 1, f"Expected 1 speed release regression, got {len(speed_release_regressions)}"
+    print(f"  - {speed_release_regressions[0]['model']}: Speed decreased from {speed_release_regressions[0]['best_historical_value']}x to {speed_release_regressions[0]['current_value']}x")
+    
+    # Test tokens release regression
+    tokens_release_regressions = detect_tokens_release_regressions(current_data, previous_data, threshold=20.0)
+    print(f"\n✓ Tokens/Sec Release Regressions Found: {len(tokens_release_regressions)}")
+    assert len(tokens_release_regressions) == 1, f"Expected 1 tokens release regression, got {len(tokens_release_regressions)}"
+    print(f"  - {tokens_release_regressions[0]['model']}: Tokens/sec decreased from {tokens_release_regressions[0]['best_historical_value']} to {tokens_release_regressions[0]['current_value']}")
+    
+    print("\n✅ TEST 4 PASSED: Release-to-release regression detection works correctly")
+    return True
+
+
+def test_slack_message_generation():
+    """Test Slack message generation"""
+    print("\n" + "="*80)
+    print("TEST 5: Slack Message Generation")
+    print("="*80)
+    
+    # Create sample regressions
+    sample_regressions = [
+        {
+            "type": "device_wer_discrepancy",
+            "metric": "WER",
+            "model": "test-model",
+            "device": "iPad Pro",
+            "current_value": 35.0,
+            "best_value": 25.0,
+            "best_device": "iPhone 15",
+            "best_os": "iOS 18",
+            "percentage_diff": 40.0
+        },
+        {
+            "type": "device_speed_discrepancy",
+            "metric": "Speed",
+            "model": "test-model",
+            "device": "iPad Pro",
+            "current_value": 60.0,
+            "best_value": 100.0,
+            "best_device": "iPhone 15",
+            "best_os": "iOS 18",
+            "percentage_diff": 40.0
+        }
+    ]
+    
+    # Generate Slack message
+    slack_payload = generate_slack_message(sample_regressions)
+    
+    assert slack_payload is not None, "Expected Slack payload to be generated"
+    assert "blocks" in slack_payload, "Expected 'blocks' in Slack payload"
+    assert len(slack_payload["blocks"]) > 0, "Expected at least one block in Slack payload"
+    
+    print(f"\n✓ Slack Message Generated Successfully")
+    print(f"  - Total blocks: {len(slack_payload['blocks'])}")
+    print(f"\n📧 Full Slack Message Payload:")
+    print("=" * 80)
+    print(json.dumps(slack_payload, indent=2))
+    print("=" * 80)
+    
+    print("\n✅ TEST 5 PASSED: Slack message generation works correctly")
+    return True
+
+
+def test_edge_cases():
+    """Test edge cases"""
+    print("\n" + "="*80)
+    print("TEST 6: Edge Cases")
+    print("="*80)
+    
+    # Test with single data point (should not trigger any regressions - no historical comparison)
+    single_current = [
+        {"model": "model-a", "device": "iPhone 15", "os": "iOS 18", "average_wer": 10.0, "speed": 100.0, "tokens_per_second": 500.0},
+    ]
+    empty_historical = []
+    
+    device_regressions = detect_device_regressions(single_current, empty_historical, threshold=20.0)
+    assert len(device_regressions) == 0, f"Expected 0 regressions with no historical data, got {len(device_regressions)}"
+    print("✓ Single data point with no historical data handled correctly (no regressions)")
+    
+    # Test with empty current data
+    empty_regressions = detect_device_regressions([], single_current, threshold=20.0)
+    assert len(empty_regressions) == 0, "Expected 0 regressions with empty current data"
+    print("✓ Empty current data handled correctly")
+    
+    # Test with missing fields (tokens_per_second missing)
+    partial_historical = [
+        {"model": "model-a", "device": "iPhone 15", "os": "iOS 18", "average_wer": 10.0, "speed": 100.0},
+        {"model": "model-a", "device": "iPad Pro", "os": "iOS 18", "average_wer": 10.5, "speed": 95.0},
+    ]
+    partial_current = [
+        {"model": "model-a", "device": "iPad Pro", "os": "iOS 18", "average_wer": 30.0, "speed": 80.0},
+    ]
+    
+    # Should still work for WER and speed
+    device_regressions = detect_device_regressions(partial_current, partial_historical, threshold=20.0)
+    print(f"✓ Partial data (missing tokens) handled correctly: {len(device_regressions)} WER regressions found")
+    
+    # Should not crash for tokens
+    tokens_regressions = detect_tokens_device_regressions(partial_current, partial_historical, threshold=20.0)
+    assert len(tokens_regressions) == 0, "Expected 0 tokens regressions when field is missing"
+    print("✓ Missing tokens_per_second field handled gracefully")
+    
+    print("\n✅ TEST 6 PASSED: Edge cases handled correctly")
+    return True
+
+
+def test_with_real_data_sample():
+    """Test with a small sample of real data to verify calculations"""
+    print("\n" + "="*80)
+    print("TEST 7: Real Data Sample Verification")
+    print("="*80)
+    
+    try:
+        # Load a sample of real data
+        real_data = load_performance_data("dashboard_data/performance_data.json")
+        
+        if len(real_data) == 0:
+            print("⚠️  No real data found, skipping this test")
+            return True
+        
+        print(f"✓ Loaded {len(real_data)} real data points")
+        
+        # Get unique models
+        models = set(entry["model"] for entry in real_data)
+        print(f"✓ Found {len(models)} unique models")
+        
+        # Split into current (last 10%) and historical (all data) for testing
+        split_point = int(len(real_data) * 0.9)
+        historical_data = real_data[:split_point] if split_point > 0 else real_data
+        current_data = real_data[split_point:] if split_point > 0 else real_data[:10]
+        
+        # Run detection on real data
+        device_regressions = detect_device_regressions(current_data, historical_data, threshold=20.0)
+        os_regressions = detect_os_regressions(current_data, historical_data, threshold=20.0)
+        speed_device_regressions = detect_speed_device_regressions(current_data, historical_data, threshold=20.0)
+        tokens_device_regressions = detect_tokens_device_regressions(current_data, historical_data, threshold=20.0)
+        
+        print(f"\n✓ Real Data Analysis:")
+        print(f"  - WER device regressions: {len(device_regressions)}")
+        print(f"  - WER OS regressions: {len(os_regressions)}")
+        print(f"  - Speed device regressions: {len(speed_device_regressions)}")
+        print(f"  - Tokens device regressions: {len(tokens_device_regressions)}")
+        
+        # Show a few examples if any found
+        if device_regressions:
+            print(f"\n  Example WER regression:")
+            r = device_regressions[0]
+            print(f"    Model: {r['model']}")
+            print(f"    Device: {r['device']} on {r['os']}")
+            print(f"    Current: {r['current_value']}% WER")
+            print(f"    Historical best: {r['best_value']}% WER")
+            print(f"    Deviation: +{r['percentage_diff']}%")
+        
+        if speed_device_regressions:
+            print(f"\n  Example Speed regression:")
+            r = speed_device_regressions[0]
+            print(f"    Model: {r['model']}")
+            print(f"    Device: {r['device']} on {r['os']}")
+            print(f"    Current: {r['current_value']}x speed")
+            print(f"    Historical best: {r['best_value']}x speed")
+            print(f"    Slower by: {r['percentage_diff']}%")
+        
+        print("\n✅ TEST 7 PASSED: Real data processed successfully")
+        return True
+        
+    except FileNotFoundError:
+        print("⚠️  dashboard_data/performance_data.json not found, skipping real data test")
+        return True
+    except Exception as e:
+        print(f"❌ Error processing real data: {e}")
+        return False
+
+
+def manual_verification_helper():
+    """Print data for manual verification"""
+    print("\n" + "="*80)
+    print("MANUAL VERIFICATION HELPER")
+    print("="*80)
+    
+    try:
+        real_data = load_performance_data("dashboard_data/performance_data.json")
+        
+        # Pick a model to analyze in detail
+        models = {}
+        for entry in real_data:
+            model = entry["model"]
+            if model not in models:
+                models[model] = []
+            models[model].append(entry)
+        
+        # Find a model with multiple entries
+        for model_name, entries in list(models.items())[:3]:  # Check first 3 models
+            if len(entries) >= 3:
+                print(f"\n📊 Model: {model_name}")
+                print(f"   Total data points: {len(entries)}")
+                
+                # Show WER stats
+                wer_values = [e["average_wer"] for e in entries]
+                print(f"\n   WER Analysis:")
+                print(f"   - Best (min): {min(wer_values):.2f}%")
+                print(f"   - Worst (max): {max(wer_values):.2f}%")
+                print(f"   - Difference: {((max(wer_values) - min(wer_values)) / min(wer_values) * 100):.1f}%")
+                
+                # Show by device
+                devices = {}
+                for entry in entries:
+                    device = entry["device"]
+                    if device not in devices:
+                        devices[device] = []
+                    devices[device].append(entry["average_wer"])
+                
+                print(f"\n   WER by Device:")
+                for device, wers in devices.items():
+                    avg_wer = sum(wers) / len(wers)
+                    num_samples = len(wers)
+                    print(f"   - {device}: {avg_wer:.2f}% avg ({num_samples} test runs)")
+                
+                # Show speed stats if available
+                if "speed" in entries[0]:
+                    speed_values = [e["speed"] for e in entries]
+                    print(f"\n   Speed Analysis:")
+                    print(f"   - Best (max): {max(speed_values):.2f}x")
+                    print(f"   - Worst (min): {min(speed_values):.2f}x")
+                    print(f"   - Difference: {((max(speed_values) - min(speed_values)) / max(speed_values) * 100):.1f}%")
+                
+                break
+        
+        print("\n" + "="*80)
+        print("Use the above data to manually verify regression detection logic")
+        print("="*80)
+        
+    except Exception as e:
+        print(f"Could not load data for manual verification: {e}")
+
+
+def run_all_tests():
+    """Run all tests"""
+    print("\n" + "="*80)
+    print("🧪 RUNNING ALL REGRESSION DETECTION TESTS")
+    print("="*80)
+    
+    tests = [
+        ("WER Detection (Synthetic)", test_wer_detection_with_synthetic_data),
+        ("Speed Detection (Synthetic)", test_speed_detection_with_synthetic_data),
+        ("Tokens Detection (Synthetic)", test_tokens_detection_with_synthetic_data),
+        ("Release Regression Detection", test_release_regression_detection),
+        ("Slack Message Generation", test_slack_message_generation),
+        ("Edge Cases", test_edge_cases),
+        ("Real Data Sample", test_with_real_data_sample),
+    ]
+    
+    passed = 0
+    failed = 0
+    
+    for test_name, test_func in tests:
+        try:
+            if test_func():
+                passed += 1
+            else:
+                failed += 1
+                print(f"\n❌ {test_name} FAILED")
+        except AssertionError as e:
+            failed += 1
+            print(f"\n❌ {test_name} FAILED: {e}")
+        except Exception as e:
+            failed += 1
+            print(f"\n❌ {test_name} ERROR: {e}")
+            import traceback
+            traceback.print_exc()
+    
+    # Print summary
+    print("\n" + "="*80)
+    print("TEST SUMMARY")
+    print("="*80)
+    print(f"✅ Passed: {passed}/{len(tests)}")
+    print(f"❌ Failed: {failed}/{len(tests)}")
+    
+    if failed == 0:
+        print("\n🎉 ALL TESTS PASSED! The implementation is working correctly.")
+        print("\nNext steps:")
+        print("1. Run manual verification helper to spot-check real data")
+        print("2. Test in a non-production environment first")
+        print("3. Monitor the first few runs carefully")
+    else:
+        print(f"\n⚠️  {failed} test(s) failed. Please review and fix issues.")
+        return False
+    
+    return True
+
+
+if __name__ == "__main__":
+    success = run_all_tests()
+    
+    # Optionally run manual verification helper
+    print("\n" + "="*80)
+    response = input("Run manual verification helper? (y/n): ")
+    if response.lower() == 'y':
+        manual_verification_helper()
+    
+    sys.exit(0 if success else 1)
+

.github/scripts/wer_regression_check.py

@@ -1,13 +1,14 @@
 #!/usr/bin/env python3
 """
-WhisperKit WER Regression Detection Script
+WhisperKit Performance Regression Detection Script
 
-This script detects significant WER (Word Error Rate) regressions across:
-- Different devices
-- OS versions  
-- Previous WhisperKit releases
+This script detects significant performance regressions per model by:
+- Tracking the best (lowest) WER for each model
+- Tracking the best (highest) speed and tokens per second for each model
+- Comparing all configurations against those best baselines
+- Alerting if any configuration deviates by > 20%
 
-If any model shows WER discrepancy > 20%, it alerts via Slack.
+If any model shows discrepancy > 20%, it alerts via Slack.
 """
 
 import json
@@ -51,166 +52,516 @@ def calculate_wer_statistics(wer_values: List[float]) -> Dict[str, float]:
     }
 
 
-def detect_device_regressions(data: List[Dict], threshold: float = 20.0) -> List[Dict]:
+def detect_device_regressions(current_data: List[Dict], all_historical_data: List[Dict], threshold: float = 20.0) -> List[Dict]:
     """
-    Detect WER regressions across different devices for each model/OS combination.
+    Detect WER regressions for devices in current release.
+    Compares current data points against historical best for each model+device combination.
     Returns list of regression alerts.
     """
     regressions = []
     
-    # Group by model and OS, then compare across devices
-    model_os_data = defaultdict(list)
-    for entry in data:
-        model_os_key = (entry["model"], entry["os"])
-        model_os_data[model_os_key].append(entry)
+    # Build historical best WER for each model+device combination
+    historical_best = {}
+    best_configs = {}
+    for entry in all_historical_data:
+        key = (entry["model"], entry["device"])
+        if key not in historical_best:
+            historical_best[key] = entry["average_wer"]
+            best_configs[key] = entry
+        elif entry["average_wer"] < historical_best[key]:
+            historical_best[key] = entry["average_wer"]
+            best_configs[key] = entry
+    
+    # Check each current data point against historical best
+    for entry in current_data:
+        key = (entry["model"], entry["device"])
+        
+        if key not in historical_best:
+            continue  # No historical data for this combination
+        
+        best_wer = historical_best[key]
+        best_config = best_configs[key]
+        current_wer = entry["average_wer"]
+        
+        if best_wer > 0:  # Avoid division by zero
+            pct_diff = (current_wer - best_wer) / best_wer * 100
+            
+            # Only flag if current is significantly worse than historical best
+            if pct_diff > threshold:
+                regressions.append({
+                    "type": "device_wer_discrepancy",
+                    "metric": "WER",
+                    "model": entry["model"],
+                    "device": entry["device"],
+                    "os": entry["os"],
+                    "current_value": round(current_wer, 2),
+                    "best_value": round(best_wer, 2),
+                    "best_device": best_config["device"],
+                    "best_os": best_config["os"],
+                    "percentage_diff": round(pct_diff, 1)
+                })
     
-    for (model, os), entries in model_os_data.items():
-        if len(entries) < 2:  # Need at least 2 data points to compare
-            continue
+    return regressions
+
+
+def detect_os_regressions(current_data: List[Dict], all_historical_data: List[Dict], threshold: float = 20.0) -> List[Dict]:
+    """
+    Detect WER regressions for OS versions in current release.
+    Compares current data points against historical best for each model+OS combination.
+    Returns list of regression alerts.
+    """
+    regressions = []
+    
+    # Build historical best WER for each model+OS combination
+    historical_best = {}
+    best_configs = {}
+    for entry in all_historical_data:
+        key = (entry["model"], entry["os"])
+        if key not in historical_best:
+            historical_best[key] = entry["average_wer"]
+            best_configs[key] = entry
+        elif entry["average_wer"] < historical_best[key]:
+            historical_best[key] = entry["average_wer"]
+            best_configs[key] = entry
+    
+    # Check each current data point against historical best
+    for entry in current_data:
+        key = (entry["model"], entry["os"])
+        
+        if key not in historical_best:
+            continue  # No historical data for this combination
+        
+        best_wer = historical_best[key]
+        best_config = best_configs[key]
+        current_wer = entry["average_wer"]
+        
+        if best_wer > 0:  # Avoid division by zero
+            pct_diff = (current_wer - best_wer) / best_wer * 100
+            
+            # Only flag if current is significantly worse than historical best
+            if pct_diff > threshold:
+                regressions.append({
+                    "type": "os_wer_discrepancy",
+                    "metric": "WER",
+                    "model": entry["model"],
+                    "device": entry["device"],
+                    "os": entry["os"],
+                    "current_value": round(current_wer, 2),
+                    "best_value": round(best_wer, 2),
+                    "best_device": best_config["device"],
+                    "best_os": best_config["os"],
+                    "percentage_diff": round(pct_diff, 1)
+                })
+    
+    return regressions
+
+
+def detect_release_regressions(current_data: List[Dict], previous_data: List[Dict], 
+                              threshold: float = 20.0) -> List[Dict]:
+    """
+    Detect WER regressions in current release for each model.
+    Compares current WER against the best (lowest) historical WER for that model.
+    Returns list of regression alerts.
+    """
+    regressions = []
+    
+    if not previous_data:
+        print("No previous release data available for comparison")
+        return regressions
+    
+    # Combine all historical data
+    all_historical = previous_data
+    
+    # Group by model
+    model_current = defaultdict(list)
+    model_historical = defaultdict(list)
+    
+    for entry in current_data:
+        model_current[entry["model"]].append(entry)
+    
+    for entry in all_historical:
+        model_historical[entry["model"]].append(entry)
+    
+    # Check each model
+    for model in model_current.keys():
+        if model not in model_historical:
+            continue  # No historical data for this model
+        
+        # Find best historical WER for this model
+        best_historical_wer = min(entry["average_wer"] for entry in model_historical[model])
+        best_config = next(e for e in model_historical[model] if e["average_wer"] == best_historical_wer)
+        
+        # Check each current configuration against best historical
+        for current_entry in model_current[model]:
+            current_wer = current_entry["average_wer"]
             
-        # Group by device
-        device_wer = defaultdict(list)
-        for entry in entries:
-            device_wer[entry["device"]].append(entry["average_wer"])
-        
-        # Calculate statistics for each device
-        device_stats = {}
-        for device, wer_values in device_wer.items():
-            device_stats[device] = calculate_wer_statistics(wer_values)
-        
-        # Find significant discrepancies between devices
-        devices = list(device_stats.keys())
-        for i in range(len(devices)):
-            for j in range(i + 1, len(devices)):
-                device_1, device_2 = devices[i], devices[j]
-                mean_1 = device_stats[device_1]["mean"]
-                mean_2 = device_stats[device_2]["mean"]
+            if best_historical_wer > 0:  # Avoid division by zero
+                pct_change = (current_wer - best_historical_wer) / best_historical_wer * 100
                 
-                # Calculate percentage difference
-                if mean_1 > 0:  # Avoid division by zero
-                    pct_diff = abs(mean_2 - mean_1) / mean_1 * 100
-                    
-                    if pct_diff > threshold:
-                        regressions.append({
-                            "type": "device_discrepancy",
-                            "model": model,
-                            "os": os,
-                            "device_1": device_1,
-                            "device_2": device_2,
-                            "wer_1": round(mean_1, 2),
-                            "wer_2": round(mean_2, 2),
-                            "percentage_diff": round(pct_diff, 1)
-                        })
+                # Only flag significant WER increases (regressions)
+                if pct_change > threshold:
+                    regressions.append({
+                        "type": "release_wer_regression",
+                        "metric": "WER",
+                        "model": model,
+                        "device": current_entry["device"],
+                        "os": current_entry["os"],
+                        "current_value": round(current_wer, 2),
+                        "best_historical_value": round(best_historical_wer, 2),
+                        "best_device": best_config["device"],
+                        "best_os": best_config["os"],
+                        "percentage_increase": round(pct_change, 1)
+                    })
     
     return regressions
 
 
-def detect_os_regressions(data: List[Dict], threshold: float = 20.0) -> List[Dict]:
+def detect_speed_device_regressions(current_data: List[Dict], all_historical_data: List[Dict], threshold: float = 20.0) -> List[Dict]:
     """
-    Detect WER regressions across different OS versions for each model/device combination.
+    Detect speed regressions for devices in current release.
+    Compares current data points against historical best for each model+device combination.
     Returns list of regression alerts.
     """
     regressions = []
     
-    # Group by model and device, then compare across OS versions
-    model_device_data = defaultdict(list)
-    for entry in data:
-        model_device_key = (entry["model"], entry["device"])
-        model_device_data[model_device_key].append(entry)
+    # Build historical best speed for each model+device combination
+    historical_best = {}
+    best_configs = {}
+    for entry in all_historical_data:
+        if "speed" not in entry:
+            continue
+        key = (entry["model"], entry["device"])
+        if key not in historical_best:
+            historical_best[key] = entry["speed"]
+            best_configs[key] = entry
+        elif entry["speed"] > historical_best[key]:
+            historical_best[key] = entry["speed"]
+            best_configs[key] = entry
+    
+    # Check each current data point against historical best
+    for entry in current_data:
+        if "speed" not in entry:
+            continue
+            
+        key = (entry["model"], entry["device"])
+        
+        if key not in historical_best:
+            continue  # No historical data for this combination
+        
+        best_speed = historical_best[key]
+        best_config = best_configs[key]
+        current_speed = entry["speed"]
+        
+        if best_speed > 0:  # Avoid division by zero
+            pct_diff = (best_speed - current_speed) / best_speed * 100
+            
+            # Only flag if current is significantly slower than historical best
+            if pct_diff > threshold:
+                regressions.append({
+                    "type": "device_speed_discrepancy",
+                    "metric": "Speed",
+                    "model": entry["model"],
+                    "device": entry["device"],
+                    "os": entry["os"],
+                    "current_value": round(current_speed, 2),
+                    "best_value": round(best_speed, 2),
+                    "best_device": best_config["device"],
+                    "best_os": best_config["os"],
+                    "percentage_diff": round(pct_diff, 1)
+                })
+    
+    return regressions
+
+
+def detect_speed_os_regressions(current_data: List[Dict], all_historical_data: List[Dict], threshold: float = 20.0) -> List[Dict]:
+    """
+    Detect speed regressions for OS versions in current release.
+    Compares current data points against historical best for each model+OS combination.
+    Returns list of regression alerts.
+    """
+    regressions = []
     
-    for (model, device), entries in model_device_data.items():
-        if len(entries) < 2:  # Need at least 2 data points to compare
+    # Build historical best speed for each model+OS combination
+    historical_best = {}
+    best_configs = {}
+    for entry in all_historical_data:
+        if "speed" not in entry:
+            continue
+        key = (entry["model"], entry["os"])
+        if key not in historical_best:
+            historical_best[key] = entry["speed"]
+            best_configs[key] = entry
+        elif entry["speed"] > historical_best[key]:
+            historical_best[key] = entry["speed"]
+            best_configs[key] = entry
+    
+    # Check each current data point against historical best
+    for entry in current_data:
+        if "speed" not in entry:
             continue
             
-        # Group by OS
-        os_wer = defaultdict(list)
-        for entry in entries:
-            os_wer[entry["os"]].append(entry["average_wer"])
-        
-        # Calculate statistics for each OS
-        os_stats = {}
-        for os, wer_values in os_wer.items():
-            os_stats[os] = calculate_wer_statistics(wer_values)
-        
-        # Find significant discrepancies between OS versions
-        os_versions = list(os_stats.keys())
-        for i in range(len(os_versions)):
-            for j in range(i + 1, len(os_versions)):
-                os_1, os_2 = os_versions[i], os_versions[j]
-                mean_1 = os_stats[os_1]["mean"]
-                mean_2 = os_stats[os_2]["mean"]
-                
-                # Calculate percentage difference
-                if mean_1 > 0:  # Avoid division by zero
-                    pct_diff = abs(mean_2 - mean_1) / mean_1 * 100
-                    
-                    if pct_diff > threshold:
-                        regressions.append({
-                            "type": "os_discrepancy",
-                            "model": model,
-                            "device": device,
-                            "os_1": os_1,
-                            "os_2": os_2,
-                            "wer_1": round(mean_1, 2),
-                            "wer_2": round(mean_2, 2),
-                            "percentage_diff": round(pct_diff, 1)
-                        })
+        key = (entry["model"], entry["os"])
+        
+        if key not in historical_best:
+            continue  # No historical data for this combination
+        
+        best_speed = historical_best[key]
+        best_config = best_configs[key]
+        current_speed = entry["speed"]
+        
+        if best_speed > 0:  # Avoid division by zero
+            pct_diff = (best_speed - current_speed) / best_speed * 100
+            
+            # Only flag if current is significantly slower than historical best
+            if pct_diff > threshold:
+                regressions.append({
+                    "type": "os_speed_discrepancy",
+                    "metric": "Speed",
+                    "model": entry["model"],
+                    "device": entry["device"],
+                    "os": entry["os"],
+                    "current_value": round(current_speed, 2),
+                    "best_value": round(best_speed, 2),
+                    "best_device": best_config["device"],
+                    "best_os": best_config["os"],
+                    "percentage_diff": round(pct_diff, 1)
+                })
     
     return regressions
 
 
-def detect_release_regressions(current_data: List[Dict], previous_data: List[Dict], 
-                              threshold: float = 20.0) -> List[Dict]:
+def detect_speed_release_regressions(current_data: List[Dict], previous_data: List[Dict], 
+                                     threshold: float = 20.0) -> List[Dict]:
     """
-    Detect WER regressions between WhisperKit releases.
+    Detect speed regressions in current release for each model.
+    Compares current speed against the best (highest) historical speed for that model.
     Returns list of regression alerts.
     """
     regressions = []
     
     if not previous_data:
-        print("No previous release data available for comparison")
         return regressions
     
-    # Create lookup dictionaries by (model, device, os)
-    current_lookup = {}
-    previous_lookup = {}
+    # Group by model
+    model_current = defaultdict(list)
+    model_historical = defaultdict(list)
     
     for entry in current_data:
-        key = (entry["model"], entry["device"], entry["os"])
-        current_lookup[key] = entry["average_wer"]
+        if "speed" in entry:
+            model_current[entry["model"]].append(entry)
     
     for entry in previous_data:
-        key = (entry["model"], entry["device"], entry["os"])
-        previous_lookup[key] = entry["average_wer"]
+        if "speed" in entry:
+            model_historical[entry["model"]].append(entry)
+    
+    # Check each model
+    for model in model_current.keys():
+        if model not in model_historical:
+            continue  # No historical data for this model
+        
+        # Find best historical speed for this model
+        best_historical_speed = max(entry["speed"] for entry in model_historical[model])
+        best_config = next(e for e in model_historical[model] if e["speed"] == best_historical_speed)
+        
+        # Check each current configuration against best historical
+        for current_entry in model_current[model]:
+            current_speed = current_entry["speed"]
+            
+            if best_historical_speed > 0:  # Avoid division by zero
+                pct_change = (best_historical_speed - current_speed) / best_historical_speed * 100
+                
+                # Only flag significant speed decreases (regressions)
+                if pct_change > threshold:
+                    regressions.append({
+                        "type": "release_speed_regression",
+                        "metric": "Speed",
+                        "model": model,
+                        "device": current_entry["device"],
+                        "os": current_entry["os"],
+                        "current_value": round(current_speed, 2),
+                        "best_historical_value": round(best_historical_speed, 2),
+                        "best_device": best_config["device"],
+                        "best_os": best_config["os"],
+                        "percentage_decrease": round(pct_change, 1)
+                    })
+    
+    return regressions
+
+
+def detect_tokens_device_regressions(current_data: List[Dict], all_historical_data: List[Dict], threshold: float = 20.0) -> List[Dict]:
+    """
+    Detect tokens per second regressions for devices in current release.
+    Compares current data points against historical best for each model+device combination.
+    Returns list of regression alerts.
+    """
+    regressions = []
+    
+    # Build historical best tokens/sec for each model+device combination
+    historical_best = {}
+    best_configs = {}
+    for entry in all_historical_data:
+        if "tokens_per_second" not in entry:
+            continue
+        key = (entry["model"], entry["device"])
+        if key not in historical_best:
+            historical_best[key] = entry["tokens_per_second"]
+            best_configs[key] = entry
+        elif entry["tokens_per_second"] > historical_best[key]:
+            historical_best[key] = entry["tokens_per_second"]
+            best_configs[key] = entry
+    
+    # Check each current data point against historical best
+    for entry in current_data:
+        if "tokens_per_second" not in entry:
+            continue
+            
+        key = (entry["model"], entry["device"])
+        
+        if key not in historical_best:
+            continue  # No historical data for this combination
+        
+        best_tokens = historical_best[key]
+        best_config = best_configs[key]
+        current_tokens = entry["tokens_per_second"]
+        
+        if best_tokens > 0:  # Avoid division by zero
+            pct_diff = (best_tokens - current_tokens) / best_tokens * 100
+            
+            # Only flag if current is significantly slower than historical best
+            if pct_diff > threshold:
+                regressions.append({
+                    "type": "device_tokens_discrepancy",
+                    "metric": "Tokens/Second",
+                    "model": entry["model"],
+                    "device": entry["device"],
+                    "os": entry["os"],
+                    "current_value": round(current_tokens, 2),
+                    "best_value": round(best_tokens, 2),
+                    "best_device": best_config["device"],
+                    "best_os": best_config["os"],
+                    "percentage_diff": round(pct_diff, 1)
+                })
     
-    # Compare common configurations
-    common_configs = set(current_lookup.keys()) & set(previous_lookup.keys())
+    return regressions
+
+
+def detect_tokens_os_regressions(current_data: List[Dict], all_historical_data: List[Dict], threshold: float = 20.0) -> List[Dict]:
+    """
+    Detect tokens per second regressions for OS versions in current release.
+    Compares current data points against historical best for each model+OS combination.
+    Returns list of regression alerts.
+    """
+    regressions = []
     
-    for config in common_configs:
-        model, device, os = config
-        current_wer = current_lookup[config]
-        previous_wer = previous_lookup[config]
+    # Build historical best tokens/sec for each model+OS combination
+    historical_best = {}
+    best_configs = {}
+    for entry in all_historical_data:
+        if "tokens_per_second" not in entry:
+            continue
+        key = (entry["model"], entry["os"])
+        if key not in historical_best:
+            historical_best[key] = entry["tokens_per_second"]
+            best_configs[key] = entry
+        elif entry["tokens_per_second"] > historical_best[key]:
+            historical_best[key] = entry["tokens_per_second"]
+            best_configs[key] = entry
+    
+    # Check each current data point against historical best
+    for entry in current_data:
+        if "tokens_per_second" not in entry:
+            continue
+            
+        key = (entry["model"], entry["os"])
+        
+        if key not in historical_best:
+            continue  # No historical data for this combination
         
-        if previous_wer > 0:  # Avoid division by zero
-            pct_change = (current_wer - previous_wer) / previous_wer * 100
+        best_tokens = historical_best[key]
+        best_config = best_configs[key]
+        current_tokens = entry["tokens_per_second"]
+        
+        if best_tokens > 0:  # Avoid division by zero
+            pct_diff = (best_tokens - current_tokens) / best_tokens * 100
             
-            # Only flag significant WER increases (regressions)
-            if pct_change > threshold:
+            # Only flag if current is significantly slower than historical best
+            if pct_diff > threshold:
                 regressions.append({
-                    "type": "release_regression",
+                    "type": "os_tokens_discrepancy",
+                    "metric": "Tokens/Second",
+                    "model": entry["model"],
+                    "device": entry["device"],
+                    "os": entry["os"],
+                    "current_value": round(current_tokens, 2),
+                    "best_value": round(best_tokens, 2),
+                    "best_device": best_config["device"],
+                    "best_os": best_config["os"],
+                    "percentage_diff": round(pct_diff, 1)
+                })
+    
+    return regressions
+
+
+def detect_tokens_release_regressions(current_data: List[Dict], previous_data: List[Dict], 
+                                      threshold: float = 20.0) -> List[Dict]:
+    """
+    Detect tokens per second regressions in current release for each model.
+    Compares current tokens/sec against the best (highest) historical tokens/sec for that model.
+    Returns list of regression alerts.
+    """
+    regressions = []
+    
+    if not previous_data:
+        return regressions
+    
+    # Group by model
+    model_current = defaultdict(list)
+    model_historical = defaultdict(list)
+    
+    for entry in current_data:
+        if "tokens_per_second" in entry:
+            model_current[entry["model"]].append(entry)
+    
+    for entry in previous_data:
+        if "tokens_per_second" in entry:
+            model_historical[entry["model"]].append(entry)
+    
+    # Check each model
+    for model in model_current.keys():
+        if model not in model_historical:
+            continue  # No historical data for this model
+        
+        # Find best historical tokens/sec for this model
+        best_historical_tokens = max(entry["tokens_per_second"] for entry in model_historical[model])
+        best_config = next(e for e in model_historical[model] if e["tokens_per_second"] == best_historical_tokens)
+        
+        # Check each current configuration against best historical
+        for current_entry in model_current[model]:
+            current_tokens = current_entry["tokens_per_second"]
+            
+            if best_historical_tokens > 0:  # Avoid division by zero
+                pct_change = (best_historical_tokens - current_tokens) / best_historical_tokens * 100
+                
+                # Only flag significant tokens/sec decreases (regressions)
+                if pct_change > threshold:
+                    regressions.append({
+                        "type": "release_tokens_regression",
+                        "metric": "Tokens/Second",
                     "model": model,
-                    "device": device,
-                    "os": os,
-                    "previous_wer": round(previous_wer, 2),
-                    "current_wer": round(current_wer, 2),
-                    "percentage_increase": round(pct_change, 1)
+                        "device": current_entry["device"],
+                        "os": current_entry["os"],
+                        "current_value": round(current_tokens, 2),
+                        "best_historical_value": round(best_historical_tokens, 2),
+                        "best_device": best_config["device"],
+                        "best_os": best_config["os"],
+                        "percentage_decrease": round(pct_change, 1)
                 })
     
     return regressions
 
 
 def generate_slack_message(regressions: List[Dict]) -> Dict:
-    """Generate Slack message payload for WER regression alerts."""
+    """Generate Slack message payload for performance regression alerts."""
     
     if not regressions:
         return None
@@ -220,7 +571,7 @@ def generate_slack_message(regressions: List[Dict]) -> Dict:
             "type": "header",
             "text": {
                 "type": "plain_text",
-                "text": "WhisperKit WER Regression Alert",
+                "text": "⚠️ WhisperKit Performance Regression Alert",
                 "emoji": True
             }
         },
@@ -229,7 +580,7 @@ def generate_slack_message(regressions: List[Dict]) -> Dict:
             "elements": [
                 {
                     "type": "mrkdwn",
-                    "text": f"*Detected {len(regressions)} significant WER regression(s)*"
+                    "text": f"*Detected {len(regressions)} significant performance regression(s)*"
                 }
             ]
         },
@@ -237,84 +588,239 @@ def generate_slack_message(regressions: List[Dict]) -> Dict:
     ]
     
     # Group regressions by type
-    device_regressions = [r for r in regressions if r["type"] == "device_discrepancy"]
-    os_regressions = [r for r in regressions if r["type"] == "os_discrepancy"]
-    release_regressions = [r for r in regressions if r["type"] == "release_regression"]
+    wer_device = [r for r in regressions if r["type"] == "device_wer_discrepancy"]
+    wer_os = [r for r in regressions if r["type"] == "os_wer_discrepancy"]
+    wer_release = [r for r in regressions if r["type"] == "release_wer_regression"]
+    
+    speed_device = [r for r in regressions if r["type"] == "device_speed_discrepancy"]
+    speed_os = [r for r in regressions if r["type"] == "os_speed_discrepancy"]
+    speed_release = [r for r in regressions if r["type"] == "release_speed_regression"]
     
-    if device_regressions:
+    tokens_device = [r for r in regressions if r["type"] == "device_tokens_discrepancy"]
+    tokens_os = [r for r in regressions if r["type"] == "os_tokens_discrepancy"]
+    tokens_release = [r for r in regressions if r["type"] == "release_tokens_regression"]
+    
+    # WER Regressions
+    if wer_device:
         blocks.append({
             "type": "section",
             "text": {
                 "type": "mrkdwn",
-                "text": "*Device Discrepancies:*"
+                "text": "*WER Device Discrepancies:*"
             }
         })
         
-        for regression in device_regressions:
+        for regression in wer_device:
             blocks.append({
                 "type": "section",
                 "text": {
                     "type": "mrkdwn",
-                    "text": f"*{regression['model']}* on {regression['os']}\n"
-                            f"• {regression['device_1']}: {regression['wer_1']}% WER\n"
-                            f"• {regression['device_2']}: {regression['wer_2']}% WER\n"
-                            f"• Difference: {regression['percentage_diff']}%"
+                    "text": f"*{regression['model']}*\n"
+                            f"• {regression['device']}: {regression['current_value']}% WER\n"
+                            f"• Best: {regression['best_value']}% WER ({regression['best_device']} on {regression['best_os']})\n"
+                            f"• Deviation: +{regression['percentage_diff']}%"
                 }
             })
     
-    if os_regressions:
-        if device_regressions:
+    if wer_os:
+        if wer_device:
             blocks.append({"type": "divider"})
         
         blocks.append({
             "type": "section",
             "text": {
                 "type": "mrkdwn",
-                "text": "*OS Version Discrepancies:*"
+                "text": "*WER OS Version Discrepancies:*"
             }
         })
         
-        for regression in os_regressions:
+        for regression in wer_os:
             blocks.append({
                 "type": "section",
                 "text": {
                     "type": "mrkdwn",
-                    "text": f"*{regression['model']}* on {regression['device']}\n"
-                            f"• {regression['os_1']}: {regression['wer_1']}% WER\n"
-                            f"• {regression['os_2']}: {regression['wer_2']}% WER\n"
-                            f"• Difference: {regression['percentage_diff']}%"
+                    "text": f"*{regression['model']}*\n"
+                            f"• {regression['os']}: {regression['current_value']}% WER\n"
+                            f"• Best: {regression['best_value']}% WER ({regression['best_device']} on {regression['best_os']})\n"
+                            f"• Deviation: +{regression['percentage_diff']}%"
                 }
             })
     
-    if release_regressions:
-        if device_regressions or os_regressions:
+    if wer_release:
+        if wer_device or wer_os:
             blocks.append({"type": "divider"})
         
         blocks.append({
             "type": "section",
             "text": {
                 "type": "mrkdwn",
-                "text": "*Release-to-Release Regressions:*"
+                "text": "*WER Release-to-Release Regressions:*"
             }
         })
         
-        for regression in release_regressions:
+        for regression in wer_release:
             blocks.append({
                 "type": "section",
                 "text": {
                     "type": "mrkdwn",
                     "text": f"*{regression['model']}* on {regression['device']} ({regression['os']})\n"
-                            f"• Previous: {regression['previous_wer']}% WER\n"
-                            f"• Current: {regression['current_wer']}% WER\n"
+                            f"• Current: {regression['current_value']}% WER\n"
+                            f"• Best Historical: {regression['best_historical_value']}% WER ({regression['best_device']} on {regression['best_os']})\n"
                             f"• Increase: +{regression['percentage_increase']}%"
                 }
             })
     
+    # Speed Regressions
+    if speed_device:
+        if wer_device or wer_os or wer_release:
+            blocks.append({"type": "divider"})
+        
+        blocks.append({
+            "type": "section",
+            "text": {
+                "type": "mrkdwn",
+                "text": "*Speed Device Discrepancies:*"
+            }
+        })
+        
+        for regression in speed_device:
+            blocks.append({
+                "type": "section",
+                "text": {
+                    "type": "mrkdwn",
+                    "text": f"*{regression['model']}*\n"
+                            f"• {regression['device']}: {regression['current_value']}x speed\n"
+                            f"• Best: {regression['best_value']}x speed ({regression['best_device']} on {regression['best_os']})\n"
+                            f"• Slower by: {regression['percentage_diff']}%"
+                }
+            })
+    
+    if speed_os:
+        if any([wer_device, wer_os, wer_release, speed_device]):
+            blocks.append({"type": "divider"})
+        
+        blocks.append({
+            "type": "section",
+            "text": {
+                "type": "mrkdwn",
+                "text": "*Speed OS Version Discrepancies:*"
+            }
+        })
+        
+        for regression in speed_os:
+            blocks.append({
+                "type": "section",
+                "text": {
+                    "type": "mrkdwn",
+                    "text": f"*{regression['model']}*\n"
+                            f"• {regression['os']}: {regression['current_value']}x speed\n"
+                            f"• Best: {regression['best_value']}x speed ({regression['best_device']} on {regression['best_os']})\n"
+                            f"• Slower by: {regression['percentage_diff']}%"
+                }
+            })
+    
+    if speed_release:
+        if any([wer_device, wer_os, wer_release, speed_device, speed_os]):
+            blocks.append({"type": "divider"})
+        
+        blocks.append({
+            "type": "section",
+            "text": {
+                "type": "mrkdwn",
+                "text": "*Speed Release-to-Release Regressions:*"
+            }
+        })
+        
+        for regression in speed_release:
+            blocks.append({
+                "type": "section",
+                "text": {
+                    "type": "mrkdwn",
+                    "text": f"*{regression['model']}* on {regression['device']} ({regression['os']})\n"
+                            f"• Current: {regression['current_value']}x speed\n"
+                            f"• Best Historical: {regression['best_historical_value']}x speed ({regression['best_device']} on {regression['best_os']})\n"
+                            f"• Slower by: {regression.get('percentage_decrease', regression.get('percentage_increase', 0))}%"
+                }
+            })
+    
+    # Tokens Per Second Regressions
+    if tokens_device:
+        if any([wer_device, wer_os, wer_release, speed_device, speed_os, speed_release]):
+            blocks.append({"type": "divider"})
+        
+        blocks.append({
+            "type": "section",
+            "text": {
+                "type": "mrkdwn",
+                "text": "*Tokens/Second Device Discrepancies:*"
+            }
+        })
+        
+        for regression in tokens_device:
+            blocks.append({
+                "type": "section",
+                "text": {
+                    "type": "mrkdwn",
+                    "text": f"*{regression['model']}*\n"
+                            f"• {regression['device']}: {regression['current_value']} tokens/sec\n"
+                            f"• Best: {regression['best_value']} tokens/sec ({regression['best_device']} on {regression['best_os']})\n"
+                            f"• Slower by: {regression['percentage_diff']}%"
+                }
+            })
+    
+    if tokens_os:
+        if any([wer_device, wer_os, wer_release, speed_device, speed_os, speed_release, tokens_device]):
+            blocks.append({"type": "divider"})
+        
+        blocks.append({
+            "type": "section",
+            "text": {
+                "type": "mrkdwn",
+                "text": "*Tokens/Second OS Version Discrepancies:*"
+            }
+        })
+        
+        for regression in tokens_os:
+            blocks.append({
+                "type": "section",
+                "text": {
+                    "type": "mrkdwn",
+                    "text": f"*{regression['model']}*\n"
+                            f"• {regression['os']}: {regression['current_value']} tokens/sec\n"
+                            f"• Best: {regression['best_value']} tokens/sec ({regression['best_device']} on {regression['best_os']})\n"
+                            f"• Slower by: {regression['percentage_diff']}%"
+                }
+            })
+    
+    if tokens_release:
+        if any([wer_device, wer_os, wer_release, speed_device, speed_os, speed_release, tokens_device, tokens_os]):
+            blocks.append({"type": "divider"})
+        
+        blocks.append({
+            "type": "section",
+            "text": {
+                "type": "mrkdwn",
+                "text": "*Tokens/Second Release-to-Release Regressions:*"
+            }
+        })
+        
+        for regression in tokens_release:
+            blocks.append({
+                "type": "section",
+                "text": {
+                    "type": "mrkdwn",
+                    "text": f"*{regression['model']}* on {regression['device']} ({regression['os']})\n"
+                            f"• Current: {regression['current_value']} tokens/sec\n"
+                            f"• Best Historical: {regression['best_historical_value']} tokens/sec ({regression['best_device']} on {regression['best_os']})\n"
+                            f"• Slower by: {regression.get('percentage_decrease', regression.get('percentage_increase', 0))}%"
+                }
+            })
+    
     return {"blocks": blocks}
 
 
-def check_wer_regressions():
-    """Main function to check for WER regressions and generate alerts."""
+def check_performance_regressions():
+    """Main function to check for performance regressions and generate alerts."""
     
     # Load version data to get commit hashes
     try:
@@ -333,7 +839,7 @@ def check_wer_regressions():
     current_commit = releases[-1] if releases else None
     previous_commit = releases[-2] if len(releases) >= 2 else None
     
-    print(f"Checking WER regressions for current commit: {current_commit}")
+    print(f"Checking performance regressions for current commit: {current_commit}")
     if previous_commit:
         print(f"Comparing against previous commit: {previous_commit}")
     
@@ -347,43 +853,70 @@ def check_wer_regressions():
     
     all_regressions = []
     
-    # Check for device discrepancies across all WhisperKit versions
-    device_regressions = detect_device_regressions(all_historical_data, threshold=20.0)
+    # WER Checks
+    print("\n=== Checking WER Regressions ===")
+    device_regressions = detect_device_regressions(current_data, all_historical_data, threshold=20.0)
     all_regressions.extend(device_regressions)
-    print(f"Found {len(device_regressions)} device discrepancies across WhisperKit versions")
+    print(f"Found {len(device_regressions)} WER device discrepancies")
     
-    # Check for OS discrepancies across all WhisperKit versions
-    os_regressions = detect_os_regressions(all_historical_data, threshold=20.0)
+    os_regressions = detect_os_regressions(current_data, all_historical_data, threshold=20.0)
     all_regressions.extend(os_regressions)
-    print(f"Found {len(os_regressions)} OS discrepancies across WhisperKit versions")
+    print(f"Found {len(os_regressions)} WER OS discrepancies")
     
-    # Check for release-to-release regressions
     release_regressions = detect_release_regressions(current_data, previous_data, threshold=20.0)
     all_regressions.extend(release_regressions)
-    print(f"Found {len(release_regressions)} release regressions")
+    print(f"Found {len(release_regressions)} WER release regressions")
+    
+    # Speed Checks
+    print("\n=== Checking Speed Regressions ===")
+    speed_device_regressions = detect_speed_device_regressions(current_data, all_historical_data, threshold=20.0)
+    all_regressions.extend(speed_device_regressions)
+    print(f"Found {len(speed_device_regressions)} speed device discrepancies")
+    
+    speed_os_regressions = detect_speed_os_regressions(current_data, all_historical_data, threshold=20.0)
+    all_regressions.extend(speed_os_regressions)
+    print(f"Found {len(speed_os_regressions)} speed OS discrepancies")
+    
+    speed_release_regressions = detect_speed_release_regressions(current_data, previous_data, threshold=20.0)
+    all_regressions.extend(speed_release_regressions)
+    print(f"Found {len(speed_release_regressions)} speed release regressions")
+    
+    # Tokens Per Second Checks
+    print("\n=== Checking Tokens/Second Regressions ===")
+    tokens_device_regressions = detect_tokens_device_regressions(current_data, all_historical_data, threshold=20.0)
+    all_regressions.extend(tokens_device_regressions)
+    print(f"Found {len(tokens_device_regressions)} tokens/sec device discrepancies")
+    
+    tokens_os_regressions = detect_tokens_os_regressions(current_data, all_historical_data, threshold=20.0)
+    all_regressions.extend(tokens_os_regressions)
+    print(f"Found {len(tokens_os_regressions)} tokens/sec OS discrepancies")
+    
+    tokens_release_regressions = detect_tokens_release_regressions(current_data, previous_data, threshold=20.0)
+    all_regressions.extend(tokens_release_regressions)
+    print(f"Found {len(tokens_release_regressions)} tokens/sec release regressions")
     
     # Generate outputs
     github_output = os.getenv("GITHUB_OUTPUT")
     if github_output:
         with open(github_output, "a") as f:
-            print(f"has_wer_regressions={'true' if all_regressions else 'false'}", file=f)
-            print(f"wer_regression_count={len(all_regressions)}", file=f)
+            print(f"has_performance_regressions={'true' if all_regressions else 'false'}", file=f)
+            print(f"performance_regression_count={len(all_regressions)}", file=f)
             
             if all_regressions:
                 slack_payload = generate_slack_message(all_regressions)
                 if slack_payload:
-                    f.write("wer_regression_slack_payload<<EOF\n")
+                    f.write("performance_regression_slack_payload<<EOF\n")
                     json.dump(slack_payload, f, indent=2)
                     f.write("\nEOF\n")
     
     # Print summary for debugging
     if all_regressions:
-        print(f"\nALERT: Found {len(all_regressions)} WER regressions!")
+        print(f"\n⚠️  ALERT: Found {len(all_regressions)} performance regressions!")
         for regression in all_regressions:
             print(f"  - {regression['type']}: {regression.get('model', 'N/A')}")
     else:
-        print("No significant WER regressions detected")
+        print("\n✅ No significant performance regressions detected")
 
 
 if __name__ == "__main__":
-    check_wer_regressions()
+    check_performance_regressions()