Add new check for wer discrepancy

.github/scripts/wer_regression_check.py

@@ -0,0 +1,389 @@
+#!/usr/bin/env python3
+"""
+WhisperKit WER Regression Detection Script
+
+This script detects significant WER (Word Error Rate) regressions across:
+- Different devices
+- OS versions  
+- Previous WhisperKit releases
+
+If any model shows WER discrepancy > 20%, it alerts via Slack.
+"""
+
+import json
+import os
+import statistics
+from collections import defaultdict
+from typing import Dict, List, Tuple, Optional
+import pandas as pd
+
+
+def load_performance_data(file_path: str, commit_hash: Optional[str] = None) -> List[Dict]:
+    """Load performance data from JSON file, optionally filtering by commit hash."""
+    data = []
+    try:
+        with open(file_path, "r") as f:
+            for line in f:
+                try:
+                    item = json.loads(line.strip())
+                    if commit_hash is None or item.get("commit_hash") == commit_hash:
+                        data.append(item)
+                except json.JSONDecodeError:
+                    continue
+    except FileNotFoundError:
+        print(f"Warning: Performance data file not found: {file_path}")
+        return []
+    
+    return data
+
+
+def calculate_wer_statistics(wer_values: List[float]) -> Dict[str, float]:
+    """Calculate WER statistics for a list of values."""
+    if not wer_values:
+        return {"mean": 0, "median": 0, "min": 0, "max": 0, "std": 0}
+    
+    return {
+        "mean": statistics.mean(wer_values),
+        "median": statistics.median(wer_values),
+        "min": min(wer_values),
+        "max": max(wer_values),
+        "std": statistics.stdev(wer_values) if len(wer_values) > 1 else 0
+    }
+
+
+def detect_device_regressions(data: List[Dict], threshold: float = 20.0) -> List[Dict]:
+    """
+    Detect WER regressions across different devices for each model/OS combination.
+    Returns list of regression alerts.
+    """
+    regressions = []
+    
+    # Group by model and OS, then compare across devices
+    model_os_data = defaultdict(list)
+    for entry in data:
+        model_os_key = (entry["model"], entry["os"])
+        model_os_data[model_os_key].append(entry)
+    
+    for (model, os), entries in model_os_data.items():
+        if len(entries) < 2:  # Need at least 2 data points to compare
+            continue
+            
+        # Group by device
+        device_wer = defaultdict(list)
+        for entry in entries:
+            device_wer[entry["device"]].append(entry["average_wer"])
+        
+        # Calculate statistics for each device
+        device_stats = {}
+        for device, wer_values in device_wer.items():
+            device_stats[device] = calculate_wer_statistics(wer_values)
+        
+        # Find significant discrepancies between devices
+        devices = list(device_stats.keys())
+        for i in range(len(devices)):
+            for j in range(i + 1, len(devices)):
+                device_1, device_2 = devices[i], devices[j]
+                mean_1 = device_stats[device_1]["mean"]
+                mean_2 = device_stats[device_2]["mean"]
+                
+                # Calculate percentage difference
+                if mean_1 > 0:  # Avoid division by zero
+                    pct_diff = abs(mean_2 - mean_1) / mean_1 * 100
+                    
+                    if pct_diff > threshold:
+                        regressions.append({
+                            "type": "device_discrepancy",
+                            "model": model,
+                            "os": os,
+                            "device_1": device_1,
+                            "device_2": device_2,
+                            "wer_1": round(mean_1, 2),
+                            "wer_2": round(mean_2, 2),
+                            "percentage_diff": round(pct_diff, 1)
+                        })
+    
+    return regressions
+
+
+def detect_os_regressions(data: List[Dict], threshold: float = 20.0) -> List[Dict]:
+    """
+    Detect WER regressions across different OS versions for each model/device combination.
+    Returns list of regression alerts.
+    """
+    regressions = []
+    
+    # Group by model and device, then compare across OS versions
+    model_device_data = defaultdict(list)
+    for entry in data:
+        model_device_key = (entry["model"], entry["device"])
+        model_device_data[model_device_key].append(entry)
+    
+    for (model, device), entries in model_device_data.items():
+        if len(entries) < 2:  # Need at least 2 data points to compare
+            continue
+            
+        # Group by OS
+        os_wer = defaultdict(list)
+        for entry in entries:
+            os_wer[entry["os"]].append(entry["average_wer"])
+        
+        # Calculate statistics for each OS
+        os_stats = {}
+        for os, wer_values in os_wer.items():
+            os_stats[os] = calculate_wer_statistics(wer_values)
+        
+        # Find significant discrepancies between OS versions
+        os_versions = list(os_stats.keys())
+        for i in range(len(os_versions)):
+            for j in range(i + 1, len(os_versions)):
+                os_1, os_2 = os_versions[i], os_versions[j]
+                mean_1 = os_stats[os_1]["mean"]
+                mean_2 = os_stats[os_2]["mean"]
+                
+                # Calculate percentage difference
+                if mean_1 > 0:  # Avoid division by zero
+                    pct_diff = abs(mean_2 - mean_1) / mean_1 * 100
+                    
+                    if pct_diff > threshold:
+                        regressions.append({
+                            "type": "os_discrepancy",
+                            "model": model,
+                            "device": device,
+                            "os_1": os_1,
+                            "os_2": os_2,
+                            "wer_1": round(mean_1, 2),
+                            "wer_2": round(mean_2, 2),
+                            "percentage_diff": round(pct_diff, 1)
+                        })
+    
+    return regressions
+
+
+def detect_release_regressions(current_data: List[Dict], previous_data: List[Dict], 
+                              threshold: float = 20.0) -> List[Dict]:
+    """
+    Detect WER regressions between WhisperKit releases.
+    Returns list of regression alerts.
+    """
+    regressions = []
+    
+    if not previous_data:
+        print("No previous release data available for comparison")
+        return regressions
+    
+    # Create lookup dictionaries by (model, device, os)
+    current_lookup = {}
+    previous_lookup = {}
+    
+    for entry in current_data:
+        key = (entry["model"], entry["device"], entry["os"])
+        current_lookup[key] = entry["average_wer"]
+    
+    for entry in previous_data:
+        key = (entry["model"], entry["device"], entry["os"])
+        previous_lookup[key] = entry["average_wer"]
+    
+    # Compare common configurations
+    common_configs = set(current_lookup.keys()) & set(previous_lookup.keys())
+    
+    for config in common_configs:
+        model, device, os = config
+        current_wer = current_lookup[config]
+        previous_wer = previous_lookup[config]
+        
+        if previous_wer > 0:  # Avoid division by zero
+            pct_change = (current_wer - previous_wer) / previous_wer * 100
+            
+            # Only flag significant WER increases (regressions)
+            if pct_change > threshold:
+                regressions.append({
+                    "type": "release_regression",
+                    "model": model,
+                    "device": device,
+                    "os": os,
+                    "previous_wer": round(previous_wer, 2),
+                    "current_wer": round(current_wer, 2),
+                    "percentage_increase": round(pct_change, 1)
+                })
+    
+    return regressions
+
+
+def generate_slack_message(regressions: List[Dict]) -> Dict:
+    """Generate Slack message payload for WER regression alerts."""
+    
+    if not regressions:
+        return None
+    
+    blocks = [
+        {
+            "type": "header",
+            "text": {
+                "type": "plain_text",
+                "text": "WhisperKit WER Regression Alert",
+                "emoji": True
+            }
+        },
+        {
+            "type": "context",
+            "elements": [
+                {
+                    "type": "mrkdwn",
+                    "text": f"*Detected {len(regressions)} significant WER regression(s)*"
+                }
+            ]
+        },
+        {"type": "divider"}
+    ]
+    
+    # Group regressions by type
+    device_regressions = [r for r in regressions if r["type"] == "device_discrepancy"]
+    os_regressions = [r for r in regressions if r["type"] == "os_discrepancy"]
+    release_regressions = [r for r in regressions if r["type"] == "release_regression"]
+    
+    if device_regressions:
+        blocks.append({
+            "type": "section",
+            "text": {
+                "type": "mrkdwn",
+                "text": "*Device Discrepancies:*"
+            }
+        })
+        
+        for regression in device_regressions:
+            blocks.append({
+                "type": "section",
+                "text": {
+                    "type": "mrkdwn",
+                    "text": f"*{regression['model']}* on {regression['os']}\n"
+                            f"• {regression['device_1']}: {regression['wer_1']}% WER\n"
+                            f"• {regression['device_2']}: {regression['wer_2']}% WER\n"
+                            f"• Difference: {regression['percentage_diff']}%"
+                }
+            })
+    
+    if os_regressions:
+        if device_regressions:
+            blocks.append({"type": "divider"})
+        
+        blocks.append({
+            "type": "section",
+            "text": {
+                "type": "mrkdwn",
+                "text": "*OS Version Discrepancies:*"
+            }
+        })
+        
+        for regression in os_regressions:
+            blocks.append({
+                "type": "section",
+                "text": {
+                    "type": "mrkdwn",
+                    "text": f"*{regression['model']}* on {regression['device']}\n"
+                            f"• {regression['os_1']}: {regression['wer_1']}% WER\n"
+                            f"• {regression['os_2']}: {regression['wer_2']}% WER\n"
+                            f"• Difference: {regression['percentage_diff']}%"
+                }
+            })
+    
+    if release_regressions:
+        if device_regressions or os_regressions:
+            blocks.append({"type": "divider"})
+        
+        blocks.append({
+            "type": "section",
+            "text": {
+                "type": "mrkdwn",
+                "text": "*Release-to-Release Regressions:*"
+            }
+        })
+        
+        for regression in release_regressions:
+            blocks.append({
+                "type": "section",
+                "text": {
+                    "type": "mrkdwn",
+                    "text": f"*{regression['model']}* on {regression['device']} ({regression['os']})\n"
+                            f"• Previous: {regression['previous_wer']}% WER\n"
+                            f"• Current: {regression['current_wer']}% WER\n"
+                            f"• Increase: +{regression['percentage_increase']}%"
+                }
+            })
+    
+    return {"blocks": blocks}
+
+
+def check_wer_regressions():
+    """Main function to check for WER regressions and generate alerts."""
+    
+    # Load version data to get commit hashes
+    try:
+        with open("dashboard_data/version.json", "r") as f:
+            version_data = json.load(f)
+    except FileNotFoundError:
+        print("Error: version.json not found")
+        return
+    
+    releases = version_data.get("releases", [])
+    if len(releases) < 1:
+        print("Not enough release data for comparison")
+        return
+    
+    # Get current and previous commit hashes
+    current_commit = releases[-1] if releases else None
+    previous_commit = releases[-2] if len(releases) >= 2 else None
+    
+    print(f"Checking WER regressions for current commit: {current_commit}")
+    if previous_commit:
+        print(f"Comparing against previous commit: {previous_commit}")
+    
+    # Load performance data - get all historical data for cross-version analysis
+    all_historical_data = load_performance_data("dashboard_data/performance_data.json")
+    current_data = load_performance_data("dashboard_data/performance_data.json", current_commit)
+    previous_data = load_performance_data("dashboard_data/performance_data.json", previous_commit) if previous_commit else []
+    
+    print(f"Loaded {len(current_data)} current data points, {len(previous_data)} previous data points")
+    print(f"Loaded {len(all_historical_data)} total historical data points for cross-version analysis")
+    
+    all_regressions = []
+    
+    # Check for device discrepancies across all WhisperKit versions
+    device_regressions = detect_device_regressions(all_historical_data, threshold=20.0)
+    all_regressions.extend(device_regressions)
+    print(f"Found {len(device_regressions)} device discrepancies across WhisperKit versions")
+    
+    # Check for OS discrepancies across all WhisperKit versions
+    os_regressions = detect_os_regressions(all_historical_data, threshold=20.0)
+    all_regressions.extend(os_regressions)
+    print(f"Found {len(os_regressions)} OS discrepancies across WhisperKit versions")
+    
+    # Check for release-to-release regressions
+    release_regressions = detect_release_regressions(current_data, previous_data, threshold=20.0)
+    all_regressions.extend(release_regressions)
+    print(f"Found {len(release_regressions)} release regressions")
+    
+    # Generate outputs
+    github_output = os.getenv("GITHUB_OUTPUT")
+    if github_output:
+        with open(github_output, "a") as f:
+            print(f"has_wer_regressions={'true' if all_regressions else 'false'}", file=f)
+            print(f"wer_regression_count={len(all_regressions)}", file=f)
+            
+            if all_regressions:
+                slack_payload = generate_slack_message(all_regressions)
+                if slack_payload:
+                    f.write("wer_regression_slack_payload<<EOF\n")
+                    json.dump(slack_payload, f, indent=2)
+                    f.write("\nEOF\n")
+    
+    # Print summary for debugging
+    if all_regressions:
+        print(f"\nALERT: Found {len(all_regressions)} WER regressions!")
+        for regression in all_regressions:
+            print(f"  - {regression['type']}: {regression.get('model', 'N/A')}")
+    else:
+        print("No significant WER regressions detected")
+
+
+if __name__ == "__main__":
+    check_wer_regressions()

.github/workflows/dataset_update.yml

@@ -146,3 +146,18 @@ jobs:
             }
         env:
           SLACK_BOT_TOKEN: ${{ secrets.SLACK_BOT_TOKEN }}
+
+      - name: Check for WER Regressions
+        if: steps.check.outputs.has_updates == 'true'
+        id: wer_check
+        run: python .github/scripts/wer_regression_check.py
+
+      - name: Alert WER Regressions
+        if: steps.check.outputs.has_updates == 'true' && steps.wer_check.outputs.has_wer_regressions == 'true'
+        uses: slackapi/slack-github-action@v1.27.0
+        with:
+          channel-id: ${{ secrets.SLACK_CHANNEL_ID }}
+          payload: |
+            ${{ steps.wer_check.outputs.wer_regression_slack_payload }}
+        env:
+          SLACK_BOT_TOKEN: ${{ secrets.SLACK_BOT_TOKEN }}

dashboard_data/support_data_3f451e1.csv

@@ -0,0 +1,23 @@
+,Model,"Apple M2 (Mac14,3)","Apple M4 (Mac16,10)","Apple M4 Pro (Mac16,8)","Apple M1 (Macmini9,1)","iPad Air (5th generation) (iPad13,16)","iPad (10th generation) (iPad13,18)","iPad mini (6th generation) (iPad14,1)","iPad Air 11-inch (M2) (iPad14,8)","iPad Air 11-inch (M3) (iPad15,3)","iPad (A16) (iPad15,7)","iPad mini (A17 Pro) (iPad16,1)","iPad Pro 11-inch (M4) (iPad16,3)","iPhone 12 mini (iPhone13,1)","iPhone 13 (iPhone14,5)","iPhone 14 (iPhone14,7)","iPhone 15 (iPhone15,4)","iPhone 16 Pro (iPhone17,1)","iPhone 16e (iPhone17,5)","iPhone 17 Pro (iPhone18,1)","iPhone 17 (iPhone18,3)"
+distil-whisper_distil-large-v3,distil-whisper_distil-large-v3,✅ macOS 26.0,✅ macOS 15.6.1,✅ macOS 15.7,⚠️ <a style='color: #3B82F6; text-decoration: underline; text-decoration-style: dotted;' href=https://huggingface.co/datasets/argmaxinc/whisperkit-evals-dataset/blob/main/benchmark_data/2025-09-20T205036_3f451e1/Macmini9%2C1_summary_2025-09-22T120503.json>macOS 26.0</a>,Not Supported,Not Supported,Not Supported,? iPadOS 26.0,? iPadOS 26.0,Not Supported,Not Supported,✅ iPadOS 26.0,Not Supported,Not Supported,Not Supported,Not Supported,Not Supported,Not Supported,⚠️ <a style='color: #3B82F6; text-decoration: underline; text-decoration-style: dotted;' href=https://huggingface.co/datasets/argmaxinc/whisperkit-evals-dataset/blob/main/benchmark_data/2025-09-20T205036_3f451e1/iPhone18%2C1_summary_2025-09-24T114907.json>iOS 26.0</a>,⚠️ <a style='color: #3B82F6; text-decoration: underline; text-decoration-style: dotted;' href=https://huggingface.co/datasets/argmaxinc/whisperkit-evals-dataset/blob/main/benchmark_data/2025-09-20T205036_3f451e1/iPhone18%2C3_summary_2025-09-24T114907.json>iOS 26.0</a>
+distil-whisper_distil-large-v3_594MB,distil-whisper_distil-large-v3_594MB,✅ macOS 26.0,✅ macOS 15.6.1,✅ macOS 15.7,✅ macOS 26.0,Not Supported,Not Supported,✅ iPadOS 18.7,? iPadOS 26.0,? iPadOS 26.0,✅ iPadOS 18.7,✅ iPadOS 18.7,? iPadOS 26.0,Not Supported,✅ iOS 18.7,✅ iOS 18.7,✅ iOS 26.0,⚠️ <a style='color: #3B82F6; text-decoration: underline; text-decoration-style: dotted;' href=https://huggingface.co/datasets/argmaxinc/whisperkit-evals-dataset/blob/main/benchmark_data/2025-09-20T205036_3f451e1/iPhone17%2C1_summary_2025-09-24T114907.json>iOS 26.0</a>,✅ iOS 18.6.2,⚠️ <a style='color: #3B82F6; text-decoration: underline; text-decoration-style: dotted;' href=https://huggingface.co/datasets/argmaxinc/whisperkit-evals-dataset/blob/main/benchmark_data/2025-09-20T205036_3f451e1/iPhone18%2C1_summary_2025-09-24T114907.json>iOS 26.0</a>,⚠️ <a style='color: #3B82F6; text-decoration: underline; text-decoration-style: dotted;' href=https://huggingface.co/datasets/argmaxinc/whisperkit-evals-dataset/blob/main/benchmark_data/2025-09-20T205036_3f451e1/iPhone18%2C3_summary_2025-09-24T114907.json>iOS 26.0</a>
+distil-whisper_distil-large-v3_turbo,distil-whisper_distil-large-v3_turbo,⚠️ <a style='color: #3B82F6; text-decoration: underline; text-decoration-style: dotted;' href=https://huggingface.co/datasets/argmaxinc/whisperkit-evals-dataset/blob/main/benchmark_data/2025-09-20T205036_3f451e1/Mac14%2C3_summary_2025-09-22T150754.json>macOS 26.0</a>,✅ macOS 15.6.1,✅ macOS 15.7,Not Supported,Not Supported,Not Supported,Not Supported,? iPadOS 26.0,? iPadOS 26.0,Not Supported,Not Supported,? iPadOS 26.0,Not Supported,Not Supported,Not Supported,Not Supported,Not Supported,Not Supported,⚠️ <a style='color: #3B82F6; text-decoration: underline; text-decoration-style: dotted;' href=https://huggingface.co/datasets/argmaxinc/whisperkit-evals-dataset/blob/main/benchmark_data/2025-09-20T205036_3f451e1/iPhone18%2C1_summary_2025-09-24T114907.json>iOS 26.0</a>,⚠️ <a style='color: #3B82F6; text-decoration: underline; text-decoration-style: dotted;' href=https://huggingface.co/datasets/argmaxinc/whisperkit-evals-dataset/blob/main/benchmark_data/2025-09-20T205036_3f451e1/iPhone18%2C3_summary_2025-09-24T114907.json>iOS 26.0</a>
+distil-whisper_distil-large-v3_turbo_600MB,distil-whisper_distil-large-v3_turbo_600MB,✅ macOS 26.0,✅ macOS 15.6.1,✅ macOS 15.7,Not Supported,Not Supported,Not Supported,✅ iPadOS 18.7,? iPadOS 26.0,? iPadOS 26.0,✅ iPadOS 18.7,✅ iPadOS 18.7,? iPadOS 26.0,Not Supported,✅ iOS 18.7,✅ iOS 18.7,✅ iOS 26.0,⚠️ <a style='color: #3B82F6; text-decoration: underline; text-decoration-style: dotted;' href=https://huggingface.co/datasets/argmaxinc/whisperkit-evals-dataset/blob/main/benchmark_data/2025-09-20T205036_3f451e1/iPhone17%2C1_summary_2025-09-24T114907.json>iOS 26.0</a>,✅ iOS 18.6.2,⚠️ <a style='color: #3B82F6; text-decoration: underline; text-decoration-style: dotted;' href=https://huggingface.co/datasets/argmaxinc/whisperkit-evals-dataset/blob/main/benchmark_data/2025-09-20T205036_3f451e1/iPhone18%2C1_summary_2025-09-24T114907.json>iOS 26.0</a>,⚠️ <a style='color: #3B82F6; text-decoration: underline; text-decoration-style: dotted;' href=https://huggingface.co/datasets/argmaxinc/whisperkit-evals-dataset/blob/main/benchmark_data/2025-09-20T205036_3f451e1/iPhone18%2C3_summary_2025-09-24T114907.json>iOS 26.0</a>
+openai_whisper-base,openai_whisper-base,✅ macOS 26.0,✅ macOS 15.6.1,✅ macOS 15.7,✅ macOS 26.0,✅ iPadOS 26.0,✅ iPadOS 18.6.2,✅ iPadOS 18.7,✅ iPadOS 26.0,✅ iPadOS 26.0,✅ iPadOS 18.7,✅ iPadOS 18.7,✅ iPadOS 26.0,✅ iOS 18.7,✅ iOS 18.7,✅ iOS 18.7,✅ iOS 26.0,✅ iOS 26.0,✅ iOS 18.6.2,✅ iOS 26.0,✅ iOS 26.0
+openai_whisper-base.en,openai_whisper-base.en,✅ macOS 26.0,✅ macOS 15.6.1,✅ macOS 15.7,✅ macOS 26.0,✅ iPadOS 26.0,✅ iPadOS 18.6.2,✅ iPadOS 18.7,✅ iPadOS 26.0,✅ iPadOS 26.0,✅ iPadOS 18.7,✅ iPadOS 18.7,✅ iPadOS 26.0,✅ iOS 18.7,✅ iOS 18.7,✅ iOS 18.7,✅ iOS 26.0,✅ iOS 26.0,✅ iOS 18.6.2,✅ iOS 26.0,✅ iOS 26.0
+openai_whisper-large-v2,openai_whisper-large-v2,✅ macOS 26.0,✅ macOS 15.6.1,✅ macOS 15.7,⚠️ <a style='color: #3B82F6; text-decoration: underline; text-decoration-style: dotted;' href=https://huggingface.co/datasets/argmaxinc/whisperkit-evals-dataset/blob/main/benchmark_data/2025-09-20T205036_3f451e1/Macmini9%2C1_summary_2025-09-22T120503.json>macOS 26.0</a>,Not Supported,Not Supported,Not Supported,? iPadOS 26.0,? iPadOS 26.0,Not Supported,Not Supported,✅ iPadOS 26.0,Not Supported,Not Supported,Not Supported,Not Supported,Not Supported,Not Supported,✅ iOS 26.0,⚠️ <a style='color: #3B82F6; text-decoration: underline; text-decoration-style: dotted;' href=https://huggingface.co/datasets/argmaxinc/whisperkit-evals-dataset/blob/main/benchmark_data/2025-09-20T205036_3f451e1/iPhone18%2C3_summary_2025-09-24T114907.json>iOS 26.0</a>
+openai_whisper-large-v2_949MB,openai_whisper-large-v2_949MB,✅ macOS 26.0,✅ macOS 15.6.1,✅ macOS 15.7,✅ macOS 26.0,Not Supported,Not Supported,✅ iPadOS 18.7,? iPadOS 26.0,? iPadOS 26.0,✅ iPadOS 18.7,✅ iPadOS 18.7,? iPadOS 26.0,Not Supported,✅ iOS 18.7,✅ iOS 18.7,✅ iOS 26.0,✅ iOS 26.0,✅ iOS 18.6.2,✅ iOS 26.0,✅ iOS 26.0
+openai_whisper-large-v2_turbo,openai_whisper-large-v2_turbo,✅ macOS 26.0,✅ macOS 15.6.1,✅ macOS 15.7,Not Supported,Not Supported,Not Supported,Not Supported,? iPadOS 26.0,? iPadOS 26.0,Not Supported,Not Supported,? iPadOS 26.0,Not Supported,Not Supported,Not Supported,Not Supported,Not Supported,Not Supported,⚠️ <a style='color: #3B82F6; text-decoration: underline; text-decoration-style: dotted;' href=https://huggingface.co/datasets/argmaxinc/whisperkit-evals-dataset/blob/main/benchmark_data/2025-09-20T205036_3f451e1/iPhone18%2C1_summary_2025-09-24T114907.json>iOS 26.0</a>,⚠️ <a style='color: #3B82F6; text-decoration: underline; text-decoration-style: dotted;' href=https://huggingface.co/datasets/argmaxinc/whisperkit-evals-dataset/blob/main/benchmark_data/2025-09-20T205036_3f451e1/iPhone18%2C3_summary_2025-09-24T114907.json>iOS 26.0</a>
+openai_whisper-large-v2_turbo_955MB,openai_whisper-large-v2_turbo_955MB,✅ macOS 26.0,✅ macOS 15.6.1,✅ macOS 15.7,Not Supported,Not Supported,Not Supported,✅ iPadOS 18.7,? iPadOS 26.0,? iPadOS 26.0,✅ iPadOS 18.7,✅ iPadOS 18.7,? iPadOS 26.0,Not Supported,✅ iOS 18.7,✅ iOS 18.7,✅ iOS 26.0,✅ iOS 26.0,✅ iOS 18.6.2,✅ iOS 26.0,✅ iOS 26.0
+openai_whisper-large-v3,openai_whisper-large-v3,✅ macOS 26.0,✅ macOS 15.6.1,✅ macOS 15.7,⚠️ <a style='color: #3B82F6; text-decoration: underline; text-decoration-style: dotted;' href=https://huggingface.co/datasets/argmaxinc/whisperkit-evals-dataset/blob/main/benchmark_data/2025-09-20T205036_3f451e1/Macmini9%2C1_summary_2025-09-22T120503.json>macOS 26.0</a>,Not Supported,Not Supported,Not Supported,? iPadOS 26.0,? iPadOS 26.0,Not Supported,Not Supported,✅ iPadOS 26.0,Not Supported,Not Supported,Not Supported,Not Supported,Not Supported,Not Supported,✅ iOS 26.0,✅ iOS 26.0
+openai_whisper-large-v3-v20240930,openai_whisper-large-v3-v20240930,✅ macOS 26.0,✅ macOS 15.6.1,✅ macOS 15.7,⚠️ <a style='color: #3B82F6; text-decoration: underline; text-decoration-style: dotted;' href=https://huggingface.co/datasets/argmaxinc/whisperkit-evals-dataset/blob/main/benchmark_data/2025-09-20T205036_3f451e1/Macmini9%2C1_summary_2025-09-22T120503.json>macOS 26.0</a>,Not Supported,Not Supported,Not Supported,? iPadOS 26.0,? iPadOS 26.0,Not Supported,Not Supported,? iPadOS 26.0,Not Supported,Not Supported,Not Supported,Not Supported,Not Supported,Not Supported,⚠️ <a style='color: #3B82F6; text-decoration: underline; text-decoration-style: dotted;' href=https://huggingface.co/datasets/argmaxinc/whisperkit-evals-dataset/blob/main/benchmark_data/2025-09-20T205036_3f451e1/iPhone18%2C1_summary_2025-09-24T114907.json>iOS 26.0</a>,⚠️ <a style='color: #3B82F6; text-decoration: underline; text-decoration-style: dotted;' href=https://huggingface.co/datasets/argmaxinc/whisperkit-evals-dataset/blob/main/benchmark_data/2025-09-20T205036_3f451e1/iPhone18%2C3_summary_2025-09-24T114907.json>iOS 26.0</a>
+openai_whisper-large-v3-v20240930_626MB,openai_whisper-large-v3-v20240930_626MB,✅ macOS 26.0,✅ macOS 15.6.1,✅ macOS 15.7,✅ macOS 26.0,Not Supported,Not Supported,✅ iPadOS 18.7,⚠️ <a style='color: #3B82F6; text-decoration: underline; text-decoration-style: dotted;' href=https://huggingface.co/datasets/argmaxinc/whisperkit-evals-dataset/blob/main/benchmark_data/2025-09-20T205036_3f451e1/iPad14%2C8_summary_2025-09-26T141736.json>iPadOS 26.0</a>,⚠️ <a style='color: #3B82F6; text-decoration: underline; text-decoration-style: dotted;' href=https://huggingface.co/datasets/argmaxinc/whisperkit-evals-dataset/blob/main/benchmark_data/2025-09-20T205036_3f451e1/iPad15%2C3_summary_2025-09-26T141736.json>iPadOS 26.0</a>,✅ iPadOS 18.7,✅ iPadOS 18.7,⚠️ <a style='color: #3B82F6; text-decoration: underline; text-decoration-style: dotted;' href=https://huggingface.co/datasets/argmaxinc/whisperkit-evals-dataset/blob/main/benchmark_data/2025-09-20T205036_3f451e1/iPad16%2C3_summary_2025-09-26T141736.json>iPadOS 26.0</a>,Not Supported,✅ iOS 18.7,✅ iOS 18.7,✅ iOS 26.0,⚠️ <a style='color: #3B82F6; text-decoration: underline; text-decoration-style: dotted;' href=https://huggingface.co/datasets/argmaxinc/whisperkit-evals-dataset/blob/main/benchmark_data/2025-09-20T205036_3f451e1/iPhone17%2C1_summary_2025-09-24T114907.json>iOS 26.0</a>,✅ iOS 18.6.2,⚠️ <a style='color: #3B82F6; text-decoration: underline; text-decoration-style: dotted;' href=https://huggingface.co/datasets/argmaxinc/whisperkit-evals-dataset/blob/main/benchmark_data/2025-09-20T205036_3f451e1/iPhone18%2C1_summary_2025-09-24T114907.json>iOS 26.0</a>,⚠️ <a style='color: #3B82F6; text-decoration: underline; text-decoration-style: dotted;' href=https://huggingface.co/datasets/argmaxinc/whisperkit-evals-dataset/blob/main/benchmark_data/2025-09-20T205036_3f451e1/iPhone18%2C3_summary_2025-09-24T114907.json>iOS 26.0</a>
+openai_whisper-large-v3-v20240930_turbo,openai_whisper-large-v3-v20240930_turbo,✅ macOS 26.0,✅ macOS 15.6.1,✅ macOS 15.7,Not Supported,Not Supported,Not Supported,Not Supported,? iPadOS 26.0,? iPadOS 26.0,Not Supported,Not Supported,? iPadOS 26.0,Not Supported,Not Supported,Not Supported,Not Supported,Not Supported,Not Supported,⚠️ <a style='color: #3B82F6; text-decoration: underline; text-decoration-style: dotted;' href=https://huggingface.co/datasets/argmaxinc/whisperkit-evals-dataset/blob/main/benchmark_data/2025-09-20T205036_3f451e1/iPhone18%2C1_summary_2025-09-24T114907.json>iOS 26.0</a>,⚠️ <a style='color: #3B82F6; text-decoration: underline; text-decoration-style: dotted;' href=https://huggingface.co/datasets/argmaxinc/whisperkit-evals-dataset/blob/main/benchmark_data/2025-09-20T205036_3f451e1/iPhone18%2C3_summary_2025-09-24T114907.json>iOS 26.0</a>
+openai_whisper-large-v3-v20240930_turbo_632MB,openai_whisper-large-v3-v20240930_turbo_632MB,✅ macOS 26.0,✅ macOS 15.6.1,✅ macOS 15.7,Not Supported,Not Supported,Not Supported,✅ iPadOS 18.7,? iPadOS 26.0,? iPadOS 26.0,✅ iPadOS 18.7,✅ iPadOS 18.7,? iPadOS 26.0,Not Supported,✅ iOS 18.7,✅ iOS 18.7,✅ iOS 26.0,⚠️ <a style='color: #3B82F6; text-decoration: underline; text-decoration-style: dotted;' href=https://huggingface.co/datasets/argmaxinc/whisperkit-evals-dataset/blob/main/benchmark_data/2025-09-20T205036_3f451e1/iPhone17%2C1_summary_2025-09-24T114907.json>iOS 26.0</a>,✅ iOS 18.6.2,⚠️ <a style='color: #3B82F6; text-decoration: underline; text-decoration-style: dotted;' href=https://huggingface.co/datasets/argmaxinc/whisperkit-evals-dataset/blob/main/benchmark_data/2025-09-20T205036_3f451e1/iPhone18%2C1_summary_2025-09-24T114907.json>iOS 26.0</a>,⚠️ <a style='color: #3B82F6; text-decoration: underline; text-decoration-style: dotted;' href=https://huggingface.co/datasets/argmaxinc/whisperkit-evals-dataset/blob/main/benchmark_data/2025-09-20T205036_3f451e1/iPhone18%2C3_summary_2025-09-24T114907.json>iOS 26.0</a>
+openai_whisper-large-v3_947MB,openai_whisper-large-v3_947MB,✅ macOS 26.0,✅ macOS 15.6.1,✅ macOS 15.7,✅ macOS 26.0,Not Supported,Not Supported,✅ iPadOS 18.7,? iPadOS 26.0,? iPadOS 26.0,✅ iPadOS 18.7,✅ iPadOS 18.7,? iPadOS 26.0,Not Supported,✅ iOS 18.7,✅ iOS 18.7,✅ iOS 26.0,✅ iOS 26.0,✅ iOS 18.6.2,? iOS 26.0,✅ iOS 26.0
+openai_whisper-large-v3_turbo,openai_whisper-large-v3_turbo,✅ macOS 26.0,✅ macOS 15.6.1,✅ macOS 15.7,Not Supported,Not Supported,Not Supported,Not Supported,? iPadOS 26.0,? iPadOS 26.0,Not Supported,Not Supported,? iPadOS 26.0,Not Supported,Not Supported,Not Supported,Not Supported,Not Supported,Not Supported,⚠️ <a style='color: #3B82F6; text-decoration: underline; text-decoration-style: dotted;' href=https://huggingface.co/datasets/argmaxinc/whisperkit-evals-dataset/blob/main/benchmark_data/2025-09-20T205036_3f451e1/iPhone18%2C1_summary_2025-09-24T114907.json>iOS 26.0</a>,⚠️ <a style='color: #3B82F6; text-decoration: underline; text-decoration-style: dotted;' href=https://huggingface.co/datasets/argmaxinc/whisperkit-evals-dataset/blob/main/benchmark_data/2025-09-20T205036_3f451e1/iPhone18%2C3_summary_2025-09-24T114907.json>iOS 26.0</a>
+openai_whisper-large-v3_turbo_954MB,openai_whisper-large-v3_turbo_954MB,✅ macOS 26.0,✅ macOS 15.6.1,✅ macOS 15.7,Not Supported,Not Supported,Not Supported,✅ iPadOS 18.7,? iPadOS 26.0,? iPadOS 26.0,✅ iPadOS 18.7,✅ iPadOS 18.7,? iPadOS 26.0,Not Supported,✅ iOS 18.7,✅ iOS 18.7,✅ iOS 26.0,⚠️ <a style='color: #3B82F6; text-decoration: underline; text-decoration-style: dotted;' href=https://huggingface.co/datasets/argmaxinc/whisperkit-evals-dataset/blob/main/benchmark_data/2025-09-20T205036_3f451e1/iPhone17%2C1_summary_2025-09-24T114907.json>iOS 26.0</a>,✅ iOS 18.6.2,⚠️ <a style='color: #3B82F6; text-decoration: underline; text-decoration-style: dotted;' href=https://huggingface.co/datasets/argmaxinc/whisperkit-evals-dataset/blob/main/benchmark_data/2025-09-20T205036_3f451e1/iPhone18%2C1_summary_2025-09-24T114907.json>iOS 26.0</a>,⚠️ <a style='color: #3B82F6; text-decoration: underline; text-decoration-style: dotted;' href=https://huggingface.co/datasets/argmaxinc/whisperkit-evals-dataset/blob/main/benchmark_data/2025-09-20T205036_3f451e1/iPhone18%2C3_summary_2025-09-24T114907.json>iOS 26.0</a>
+openai_whisper-small,openai_whisper-small,✅ macOS 26.0,✅ macOS 15.6.1,✅ macOS 15.7,✅ macOS 26.0,✅ iPadOS 26.0,✅ iPadOS 18.6.2,✅ iPadOS 18.7,✅ iPadOS 26.0,✅ iPadOS 26.0,✅ iPadOS 18.7,✅ iPadOS 18.7,✅ iPadOS 26.0,✅ iOS 18.7,✅ iOS 18.7,✅ iOS 18.7,✅ iOS 26.0,✅ iOS 26.0,✅ iOS 18.6.2,✅ iOS 26.0,✅ iOS 26.0
+openai_whisper-small.en,openai_whisper-small.en,✅ macOS 26.0,✅ macOS 15.6.1,✅ macOS 15.7,✅ macOS 26.0,✅ iPadOS 26.0,✅ iPadOS 18.6.2,✅ iPadOS 18.7,✅ iPadOS 26.0,✅ iPadOS 26.0,✅ iPadOS 18.7,✅ iPadOS 18.7,✅ iPadOS 26.0,✅ iOS 18.7,✅ iOS 18.7,✅ iOS 18.7,✅ iOS 26.0,✅ iOS 26.0,✅ iOS 18.6.2,✅ iOS 26.0,✅ iOS 26.0
+openai_whisper-tiny,openai_whisper-tiny,✅ macOS 26.0,✅ macOS 15.6.1,✅ macOS 15.7,✅ macOS 26.0,✅ iPadOS 26.0,✅ iPadOS 18.6.2,✅ iPadOS 18.7,✅ iPadOS 26.0,✅ iPadOS 26.0,✅ iPadOS 18.7,✅ iPadOS 18.7,✅ iPadOS 26.0,✅ iOS 18.7,✅ iOS 18.7,✅ iOS 18.7,✅ iOS 26.0,✅ iOS 26.0,✅ iOS 18.6.2,✅ iOS 26.0,✅ iOS 26.0
+openai_whisper-tiny.en,openai_whisper-tiny.en,✅ macOS 26.0,✅ macOS 15.6.1,✅ macOS 15.7,✅ macOS 26.0,✅ iPadOS 26.0,✅ iPadOS 18.6.2,✅ iPadOS 18.7,✅ iPadOS 26.0,✅ iPadOS 26.0,✅ iPadOS 18.7,✅ iPadOS 18.7,✅ iPadOS 26.0,✅ iOS 18.7,✅ iOS 18.7,✅ iOS 18.7,✅ iOS 26.0,✅ iOS 26.0,✅ iOS 18.6.2,✅ iOS 26.0,✅ iOS 26.0

dashboard_data/test_coverage_3f451e1.json

@@ -0,0 +1,230 @@
+{
+  "commit_hash": "3f451e1",
+  "total_devices": 137,
+  "tested_devices": 94,
+  "skipped_devices": 45,
+  "coverage_percentage": 68.61313868613139,
+  "tested_device_list": [
+    "iPhone15,2",
+    "iPad14,5",
+    "MacBookAir10,1",
+    "iPad15,8",
+    "iPad14,2",
+    "iPad15,5",
+    "iPad13,11",
+    "iMac21,2",
+    "iPhone15,5",
+    "Mac14,4",
+    "Mac16,9",
+    "iPad15,7",
+    "iPhone18,3",
+    "iPhone17,1",
+    "Mac16,13",
+    "iPad14,4",
+    "iPhone13,4",
+    "Mac14,8",
+    "Mac14,10",
+    "Mac16,8",
+    "iPad13,16",
+    "iPad14,1",
+    "iPad13,5",
+    "iPhone14,5",
+    "iPhone14,8",
+    "iPad13,7",
+    "iPad16,5",
+    "iPhone17,3",
+    "Mac16,2",
+    "iPad16,6",
+    "iPad16,3",
+    "MacBookPro18,4",
+    "Mac14,7",
+    "iPad13,10",
+    "iPad13,17",
+    "Mac14,14",
+    "Mac16,3",
+    "iPad14,9",
+    "Mac16,1",
+    "iPad16,2",
+    "iPhone13,1",
+    "Mac14,5",
+    "iPhone15,3",
+    "MacBookPro18,1",
+    "iPad15,4",
+    "Mac14,6",
+    "Mac16,12",
+    "iPad14,6",
+    "Mac14,13",
+    "MacBookPro18,3",
+    "Mac16,10",
+    "iPad13,8",
+    "iPhone17,4",
+    "iPhone13,3",
+    "Mac16,11",
+    "iPhone14,6",
+    "iPhone13,2",
+    "Mac13,1",
+    "Macmini9,1",
+    "iPhone18,1",
+    "iPad13,19",
+    "iPad13,9",
+    "iPad14,8",
+    "Mac14,2",
+    "iPad14,11",
+    "iPhone15,4",
+    "MacBookPro17,1",
+    "Mac14,9",
+    "Mac16,7",
+    "iPad16,1",
+    "Mac13,2",
+    "iPhone14,3",
+    "iPad13,1",
+    "Mac16,6",
+    "Mac16,5",
+    "iPhone17,2",
+    "iPhone14,4",
+    "iPad15,3",
+    "Mac14,3",
+    "MacBookPro18,2",
+    "iPad14,3",
+    "Mac14,15",
+    "iPad13,4",
+    "iPad14,10",
+    "iPhone14,7",
+    "iPad15,6",
+    "iPad13,2",
+    "Mac14,12",
+    "iPad16,4",
+    "iPad13,6",
+    "iMac21,1",
+    "iPad13,18",
+    "iPhone14,2",
+    "iPhone17,5"
+  ],
+  "skipped_device_list": [
+    "iPhone12,3",
+    "iPhone16,2",
+    "iPhone11,6",
+    "iPhone11,2",
+    "iPad8,3",
+    "Mac15,12",
+    "iPad11,4",
+    "Mac15,9",
+    "Mac15,11",
+    "iPhone12,4",
+    "iPad8,6",
+    "Mac15,7",
+    "iPad11,1",
+    "Mac15,8",
+    "iPad8,9",
+    "iPad11,7",
+    "Mac15,5",
+    "iPad8,8",
+    "Mac15,3",
+    "iPhone11,4",
+    "iPad8,12",
+    "iPad11,2",
+    "iPad8,10",
+    "iPad8,7",
+    "iPhone12,5",
+    "iPhone11,5",
+    "iPad8,1",
+    "Mac15,4",
+    "iPad11,6",
+    "iPad8,5",
+    "iPhone12,8",
+    "iPad8,4",
+    "iPhone12,1",
+    "iPhone16,1",
+    "iPhone12,2",
+    "Mac15,10",
+    "iPad8,2",
+    "iPhone11,3",
+    "iPad8,11",
+    "iPad12,1",
+    "Mac15,13",
+    "iPad11,3",
+    "iPad12,2",
+    "Mac15,6",
+    "iPhone11,8"
+  ],
+  "tested_os_versions": [
+    "iPadOS_18.7",
+    "iOS_18.6.2",
+    "iOS_26.0",
+    "iPadOS_26.0",
+    "macOS_15.7",
+    "iOS_18.7",
+    "iPadOS_18.6.2",
+    "macOS_15.7.0",
+    "macOS_15.6.1",
+    "macOS_26.0",
+    "macOS_26.0.0"
+  ],
+  "has_target_os_coverage": false,
+  "covered_target_versions": [
+    "macOS 15",
+    "macOS 26",
+    "iOS 18",
+    "iOS 26"
+  ],
+  "missing_target_versions": [
+    "macOS 14",
+    "iOS 17"
+  ],
+  "has_target_chip_coverage": false,
+  "platform_chip_coverage": {
+    "iPad": {
+      "total_chips": 8,
+      "tested_chips": 8,
+      "coverage_percentage": 100.0,
+      "covered_chips": [
+        "A14",
+        "A15",
+        "A16",
+        "A17 Pro",
+        "M1",
+        "M2",
+        "M3",
+        "M4"
+      ],
+      "missing_chips": []
+    },
+    "iPhone": {
+      "total_chips": 6,
+      "tested_chips": 5,
+      "coverage_percentage": 83.33333333333334,
+      "covered_chips": [
+        "A14",
+        "A15",
+        "A16",
+        "A18",
+        "A18 Pro"
+      ],
+      "missing_chips": [
+        "A17 Pro"
+      ]
+    },
+    "Mac": {
+      "total_chips": 4,
+      "tested_chips": 3,
+      "coverage_percentage": 75.0,
+      "covered_chips": [
+        "M1",
+        "M2",
+        "M4"
+      ],
+      "missing_chips": [
+        "M3"
+      ]
+    }
+  },
+  "missing_target_chips": {
+    "iPad": [],
+    "iPhone": [
+      "A17 Pro"
+    ],
+    "Mac": [
+      "M3"
+    ]
+  }
+}