Update app.py

app.py

@@ -3,7 +3,7 @@ import os
 import json
 from typing import List, Tuple
 
-os.environ["GRADIO_LANGUAGE"] = "en"   
+os.environ["GRADIO_LANGUAGE"] = "en"
 
 RESULT_DIR = os.environ.get("MOECAP_RESULT_DIR")
 if not RESULT_DIR:
@@ -30,11 +30,13 @@ def json_to_row(path: str, metrics: dict) -> dict:
 
     dataset = metrics.get("dataset", "gsm8k")
 
-    method = metrics.get("method", "")             
-    precision = metrics.get("precision", "")       
-    gsm8k_e2e = metrics.get("gsm8k_e2e_s", None)   
-    gsm8k_bs = metrics.get("gsm8k_bs", None)       
-    gsm8k_gpu = metrics.get("gpu_type", "")        
+    method = metrics.get("method", "")
+    precision = metrics.get("precision", "")
+    model_type = metrics.get("model_type", "")
+
+    gsm8k_e2e = metrics.get("gsm8k_e2e_s", None)
+    gsm8k_bs = metrics.get("gsm8k_bs", None)
+    gsm8k_gpu = metrics.get("gpu_type", "")
 
     em = metrics.get("exact_match")
     correct = metrics.get("correct")
@@ -42,7 +44,7 @@ def json_to_row(path: str, metrics: dict) -> dict:
     if isinstance(correct, (int, float)) and isinstance(total, (int, float)) and total > 0:
         acc = correct / total
     else:
-        acc = em  
+        acc = em
 
     def pct(x):
         return round(x * 100, 2) if isinstance(x, (int, float)) else None
@@ -57,6 +59,7 @@ def json_to_row(path: str, metrics: dict) -> dict:
         "Model": model_cell,
         "Dataset": dataset,
         "Method": method,
+        "Model type": model_type,
         "Precision": precision,
         "GSM8K<br>E2E(s)": f2(gsm8k_e2e),
         "GSM8K<br>bs": gsm8k_bs,
@@ -64,12 +67,10 @@ def json_to_row(path: str, metrics: dict) -> dict:
         "GSM8K<br>Accuracy(%)": pct(acc),
         "GSM8K<br>Decoding T/s": f2(metrics.get("decoding_throughput")),
         "GSM8K<br>Prefill T/s": f2(metrics.get("prefill_tp")),
-
         "GSM8K<br>Prefill<br>S-MBU(%)": pct(metrics.get("prefill_smbu")),
         "GSM8K<br>Prefill<br>S-MFU(%)": pct(metrics.get("prefill_smfu")),
         "GSM8K<br>Decoding<br>S-MBU(%)": pct(metrics.get("decoding_smbu")),
         "GSM8K<br>Decoding<br>S-MFU(%)": pct(metrics.get("decoding_smfu")),
-
         "TTFT(s)": f2(metrics.get("ttft")),
         "TPOT(s)": f2(metrics.get("tpot")),
     }
@@ -148,19 +149,24 @@ def build_leaderboard_from_files(files: List[gr.File], prev_rows: list | None =
     return summary_md, table_html, all_rows
 
 
-
-def load_from_dir(dir_path: str, selected_tasks: List[str] | None = None, force_refresh: bool = False):
-
+def load_from_dir(
+    dir_path: str,
+    selected_tasks: List[str] | None = None,
+    selected_frameworks: List[str] | None = None,
+    selected_model_types: List[str] | None = None,
+    selected_precisions: List[str] | None = None,
+    force_refresh: bool = False,
+):
     try:
         pattern = f"hf://datasets/{dir_path}/**/*.json"
         dl_mode = "force_redownload" if force_refresh else None
 
         print(f"Fetching from {pattern} (mode={dl_mode})...")
         ds = load_dataset(
-            "json", 
-            data_files={"train": pattern}, 
+            "json",
+            data_files={"train": pattern},
             split="train",
-            download_mode=dl_mode
+            download_mode=dl_mode,
         )
     except Exception as e:
         empty_html = "<p>No files loaded or Dataset not found.</p>"
@@ -183,9 +189,22 @@ def load_from_dir(dir_path: str, selected_tasks: List[str] | None = None, force_
     # Dataset filter
     if selected_tasks:
         df = df[df["Dataset"].isin(selected_tasks)]
-        if df.empty:
-            empty_html = "<p>No records found.</p>"
-            return f"No records found in dataset `{dir_path}` for tasks {selected_tasks}.", empty_html
+
+    # Inference framework filter (Method)
+    if selected_frameworks:
+        df = df[df["Method"].isin(selected_frameworks)]
+
+    # Model type filter
+    if selected_model_types:
+        df = df[df["Model type"].isin(selected_model_types)]
+
+    # Precision filter
+    if selected_precisions:
+        df = df[df["Precision"].isin(selected_precisions)]
+
+    if df.empty:
+        empty_html = "<p>No records found.</p>"
+        return f"No records found in dataset `{dir_path}` after filtering.", empty_html
 
     raw_models = set()
     for cell in df["Model"].tolist():
@@ -217,8 +236,21 @@ def load_from_dir(dir_path: str, selected_tasks: List[str] | None = None, force_
     return summary_md, table_html
 
 
-def auto_refresh_from_dir(dir_path: str, selected_tasks: List[str] | None = None):
-    return load_from_dir(dir_path, selected_tasks=selected_tasks, force_refresh=True)
+def auto_refresh_from_dir(
+    dir_path: str,
+    selected_tasks: List[str] | None = None,
+    selected_frameworks: List[str] | None = None,
+    selected_model_types: List[str] | None = None,
+    selected_precisions: List[str] | None = None,
+):
+    return load_from_dir(
+        dir_path,
+        selected_tasks=selected_tasks,
+        selected_frameworks=selected_frameworks,
+        selected_model_types=selected_model_types,
+        selected_precisions=selected_precisions,
+        force_refresh=True,
+    )
 
 
 # Gradio UI
@@ -249,14 +281,10 @@ def build_app() -> gr.Blocks:
                     "- Mathematics Problem-Solving Performance — "
                     "[**GSM8K**](https://arxiv.org/abs/2110-14168)\n\n"
                     "### Columns and Metrics\n"
-                    "- Model  \n"
-                    "- Dataset  \n"
-                    "- Method  \n"
-                    "- Precision  \n"
-                    "- GSM8K E2E (s)  \n"
-                    "- GSM8K Batch Size  \n"
+                    "- End-to-End Latency (s)  \n"
+                    "- Batch Size  \n"
                     "- GPU Type  \n"
-                    "- GSM8K Accuracy (%)  \n"
+                    "- Accuracy (%)  \n"
                     "- Decoding Throughput (tokens/s)  \n"
                     "- Prefill Throughput (tokens/s)  \n"
                     "- Prefill S-MBU (%)  \n"
@@ -269,27 +297,71 @@ def build_app() -> gr.Blocks:
 
             with gr.Column(scale=1):
                 dir_path = gr.Textbox(
-                    label="HF Dataset Repo ID", 
-                    value=RESULT_DIR, 
+                    label="HF Dataset Repo ID",
+                    value=RESULT_DIR,
                     lines=1,
-                    placeholder="username/dataset-name"
+                    placeholder="username/dataset-name",
                 )
-                #  Tasks filter
+                # 1) Tasks filter
                 task_filter = gr.CheckboxGroup(
                     label="Tasks",
                     choices=["gsm8k", "arena_hard", "mmlu", "NuminaMath"],
                     value=["gsm8k", "arena_hard", "mmlu", "NuminaMath"],
                 )
-
-                
+                # 2) Inference frameworks filter
+                framework_filter = gr.CheckboxGroup(
+                    label="Inference frameworks",
+                    choices=["sglang", "vllm"],
+                    value=["sglang", "vllm"],
+                )
+                # 3) Model types filter
+                model_type_filter = gr.CheckboxGroup(
+                    label="Model types",
+                    choices=["instruct", "thinking"],
+                    value=["instruct", "thinking"],
+                )
+                # 4) Precision filter
+                precision_filter = gr.CheckboxGroup(
+                    label="Precision",
+                    choices=["bfloat16", "fp8"],
+                    value=["bfloat16", "fp8"],
+                )
+                load_dir_button = gr.Button("Load from Dataset")
 
         summary_output = gr.Markdown(label="Directory Summary")
         leaderboard_output = gr.HTML(label="Directory Metrics")
 
+        load_dir_button.click(
+            fn=load_from_dir,
+            inputs=[dir_path, task_filter, framework_filter, model_type_filter, precision_filter],
+            outputs=[summary_output, leaderboard_output],
+        )
+
+        task_filter.change(
+            fn=load_from_dir,
+            inputs=[dir_path, task_filter, framework_filter, model_type_filter, precision_filter],
+            outputs=[summary_output, leaderboard_output],
+        )
+        framework_filter.change(
+            fn=load_from_dir,
+            inputs=[dir_path, task_filter, framework_filter, model_type_filter, precision_filter],
+            outputs=[summary_output, leaderboard_output],
+        )
+        model_type_filter.change(
+            fn=load_from_dir,
+            inputs=[dir_path, task_filter, framework_filter, model_type_filter, precision_filter],
+            outputs=[summary_output, leaderboard_output],
+        )
+        precision_filter.change(
+            fn=load_from_dir,
+            inputs=[dir_path, task_filter, framework_filter, model_type_filter, precision_filter],
+            outputs=[summary_output, leaderboard_output],
+        )
+
         timer = gr.Timer(10.0)
         timer.tick(
             fn=auto_refresh_from_dir,
-            inputs=[dir_path, task_filter],
+            inputs=[dir_path, task_filter, framework_filter, model_type_filter, precision_filter],
             outputs=[summary_output, leaderboard_output],
         )