add: simple leaderboard

simple_leaderboard.py

@@ -0,0 +1,131 @@
+import os
+import glob
+import json
+import pandas as pd
+import gradio as gr
+
+
+ROOT_DIR = os.path.dirname(os.path.abspath(__file__))
+EVAL_RESULTS_DIR = os.path.join(ROOT_DIR, "eval-results")
+
+
+def load_results(results_dir: str) -> pd.DataFrame:
+    rows = []
+    all_dataset_keys = set()
+    
+    if not os.path.isdir(results_dir):
+        return pd.DataFrame(columns=["Model", "Avg PER", "Avg Duration (s)"])
+
+    # First pass: collect all dataset keys from all files
+    for path in glob.glob(os.path.join(results_dir, "*.json")):
+        try:
+            with open(path, "r", encoding="utf-8") as f:
+                data = json.load(f)
+            res = data.get("results", {})
+            all_dataset_keys.update(res.keys())
+        except Exception:
+            continue
+
+    # Use dataset keys directly as display names
+    dataset_display_names = {key: key for key in all_dataset_keys}
+
+    # Second pass: extract data
+    for path in glob.glob(os.path.join(results_dir, "*.json")):
+        try:
+            with open(path, "r", encoding="utf-8") as f:
+                data = json.load(f)
+            cfg = data.get("config", {})
+            res = data.get("results", {})
+
+            model_name = cfg.get("model_name", "unknown")
+            
+            # Extract PER for each dataset dynamically
+            per_values = {}
+            dur_values = []
+            
+            for dataset_key in all_dataset_keys:
+                dataset_data = res.get(dataset_key, {})
+                per_value = dataset_data.get("per") if dataset_data else None
+                dur_value = dataset_data.get("avg_duration") if dataset_data else None
+                
+                display_name = dataset_display_names[dataset_key]
+                per_values[f"PER {display_name}"] = per_value
+                
+                if dur_value is not None:
+                    dur_values.append(dur_value)
+            
+            # Calculate average PER across all datasets
+            per_vals = [v for v in per_values.values() if v is not None]
+            avg_per = sum(per_vals) / len(per_vals) if per_vals else None
+            
+            # Calculate average duration
+            avg_dur = sum(dur_values) / len(dur_values) if dur_values else None
+
+            row = {
+                "Model": model_name,
+                "Avg PER": avg_per,
+                "Avg Duration (s)": avg_dur,
+                "_file": os.path.basename(path),
+            }
+            row.update(per_values)
+            rows.append(row)
+            
+        except Exception:
+            continue
+
+    df = pd.DataFrame(rows)
+    if df.empty:
+        # Create default columns based on discovered datasets
+        default_cols = ["Model", "Avg PER", "Avg Duration (s)"]
+        for key in sorted(all_dataset_keys):
+            display_name = dataset_display_names[key]
+            default_cols.insert(-2, f"PER {display_name}")
+        return pd.DataFrame(columns=default_cols)
+    
+    df = df.sort_values(by=["Avg PER"], ascending=True, na_position="last")
+    return df.reset_index(drop=True)
+
+
+def build_interface():
+    with gr.Blocks() as demo:
+        gr.Markdown("# Simple Phoneme Leaderboard")
+        info = gr.Markdown(f"Results directory: `{EVAL_RESULTS_DIR}`")
+        
+        # Get initial data to determine columns dynamically
+        initial_df = load_results(EVAL_RESULTS_DIR)
+        if not initial_df.empty:
+            headers = list(initial_df.columns)
+            # Remove internal columns
+            headers = [h for h in headers if not h.startswith('_')]
+        else:
+            headers = ["Model", "Avg PER", "Avg Duration (s)"]
+        
+        table = gr.Dataframe(headers=headers, row_count=5)
+
+        def refresh():
+            df = load_results(EVAL_RESULTS_DIR)
+            if df.empty:
+                return df
+            
+            # Get the column order from the dataframe
+            cols = [c for c in df.columns if not c.startswith('_')]
+            
+            # Ensure all columns exist for the dataframe component
+            for c in cols:
+                if c not in df.columns:
+                    df[c] = None
+            return df[cols].round(3)
+
+        btn = gr.Button("Refresh")
+        btn.click(fn=refresh, outputs=table)
+
+        # Auto-load on start
+        table.value = refresh()
+    return demo
+
+
+if __name__ == "__main__":
+    demo = build_interface()
+    demo.queue().launch()
+
+

src/phoneme_eval.py

@@ -78,35 +78,58 @@ def benchmark_dataset(dataset):
 
 from datasets import load_dataset, Audio
 
+DATASET_LIST = [
+    "mirfan899/phoneme_asr",
+    "mirfan899/kids_phoneme_md",
+]
 
 def main():
-    dataset = load_dataset("mirfan899/phoneme_asr", split="train")
-    # Disable automatic audio decoding to avoid torchcodec requirement
-    dataset = dataset.cast_column("audio", Audio(decode=False))
     field = "phonetic"
 
-    unique_texts = dataset.unique(field)
-    print("Unique phonetic strings:", len(unique_texts))
+    # Collect per-model metrics across datasets
+    per_model_results = {}
 
-    dataset_unique = dataset.filter(lambda x: x[field] in unique_texts)
+    for dataset_name in DATASET_LIST:
+        try:
+            dataset = load_dataset(dataset_name, split="train")
+        except Exception as e:
+            print(f"[warn] skip dataset {dataset_name}: {e}")
+            continue
 
-    def is_valid(example):
-        phoneme_tokens = example[field].split()
-        return len(phoneme_tokens) >= 10
+        try:
+            dataset = dataset.cast_column("audio", Audio(decode=False))
+        except Exception:
+            pass
 
-    dataset_filtered = dataset_unique.filter(is_valid)
+        unique_texts = dataset.unique(field)
+        print("Unique phonetic strings (", dataset_name, "):", len(unique_texts))
 
-    dataset_final = dataset_filtered.shuffle(seed=42).select(range(min(100, len(dataset_filtered))))
+        dataset_unique = dataset.filter(lambda x: x[field] in unique_texts)
 
-    print(dataset_final)
-    print("Final size:", len(dataset_final))
-    full_results, avg_stats = benchmark_dataset(dataset_final.select(range(10)))
+        def is_valid(example):
+            phoneme_tokens = example[field].split()
+            return len(phoneme_tokens) >= 10
 
-    print("Average Statistic per model:")
-    print(avg_stats)
+        dataset_filtered = dataset_unique.filter(is_valid)
+        dataset_final = dataset_filtered.shuffle(seed=42).select(range(min(100, len(dataset_filtered))))
 
-    # Optional: inspect detailed results
-    print(full_results.head())
+        print(dataset_final)
+        print("Final size:", len(dataset_final))
+
+        full_results, avg_stats = benchmark_dataset(dataset_final.select(range(min(10, len(dataset_final)))))
+        print("Average Statistic per model (", dataset_name, "):")
+        print(avg_stats)
+
+        # Use dataset name as key (extract the actual name part)
+        dataset_key = dataset_name.split("/")[-1]  # Get the last part after the slash
+        for _, row in avg_stats.iterrows():
+            model_name = str(row["model"]).replace(" ", "-")
+            per = float(row["Average PER"]) if row["Average PER"] is not None else None
+            avg_dur = float(row["Average Duration (s)"]) if row["Average Duration (s)"] is not None else None
+
+            if model_name not in per_model_results:
+                per_model_results[model_name] = {}
+            per_model_results[model_name][dataset_key] = {"per": per, "avg_duration": avg_dur}
 
     # Save results for leaderboard consumption (one JSON per model)
     import json, os, time
@@ -114,25 +137,16 @@ def main():
     os.makedirs(results_dir, exist_ok=True)
 
     timestamp = int(time.time())
-    for _, row in avg_stats.iterrows():
-        model_name = str(row["model"]).replace(" ", "-")
+    for model_name, task_results in per_model_results.items():
         org_model = f"local/{model_name}"
-        per = float(row["Average PER"]) if row["Average PER"] is not None else None
-        avg_dur = float(row["Average Duration (s)"]) if row["Average Duration (s)"] is not None else None
-
         payload = {
             "config": {
                 "model_name": org_model,
                 "model_dtype": "float32",
                 "model_sha": ""
             },
-            "results": {
-                # Populate both keys expected by Tasks to avoid NaNs in the leaderboard
-                "phoneme_dev": {"per": per, "avg_duration": avg_dur},
-                "phoneme_test": {"per": per, "avg_duration": avg_dur}
-            }
+            "results": task_results
         }
-
         out_path = os.path.join(results_dir, f"results_{timestamp}_{model_name}.json")
         with open(out_path, "w", encoding="utf-8") as f:
             json.dump(payload, f, ensure_ascii=False, indent=2)