"""
Synthefy MUSEval Leaderboard - Main Gradio Application
Following GIFT-Eval import structure with custom layout
"""
import gradio as gr
import pandas as pd
# Optional imports for production features
try:
from apscheduler.schedulers.background import BackgroundScheduler
SCHEDULER_AVAILABLE = True
except ImportError:
SCHEDULER_AVAILABLE = False
print("Warning: apscheduler not available, scheduler features disabled")
try:
from huggingface_hub import snapshot_download
HUB_AVAILABLE = True
except ImportError:
HUB_AVAILABLE = False
print("Warning: huggingface_hub not available, hub features disabled")
from src.about import (
CITATION_BUTTON_LABEL,
CITATION_BUTTON_TEXT,
EVALUATION_QUEUE_TEXT,
INTRODUCTION_TEXT,
BENCHMARKS_TEXT,
TITLE,
)
from src.display.css_html_js import custom_css
from src.display.utils import (
BENCHMARK_COLS,
EVAL_COLS,
EVAL_TYPES,
ModelInfoColumn,
ModelType,
fields,
WeightType,
Precision
)
from src.envs import API, EVAL_REQUESTS_PATH, EVAL_RESULTS_PATH, QUEUE_REPO, REPO_ID, RESULTS_REPO, TOKEN
from src.populate import get_evaluation_queue_df, get_leaderboard_df, get_model_info_df, get_merged_df
from src.utils import norm_sNavie, pivot_df, get_grouped_dfs, pivot_existed_df, rename_metrics, format_df
from src.load_results import (
load_results_with_metadata,
create_overall_table,
create_html_table,
create_html_table_from_df,
get_filter_options,
get_model_metadata,
create_model_metadata_display,
get_overall_summary,
sort_table_by_column,
get_available_models
)
def create_model_buttons():
"""Create buttons for each model that can trigger Model Inspector updates"""
from src.load_results import get_available_models
models = get_available_models()
buttons = []
for model in models:
btn = gr.Button(
value=model,
variant="secondary",
size="sm",
scale=0.5
)
buttons.append(btn)
return buttons
def restart_space():
API.restart_space(repo_id=REPO_ID)
def create_leaderboard_interface():
"""Create the main leaderboard interface"""
demo = gr.Blocks(css=custom_css)
with demo:
gr.HTML(TITLE)
# Minimizable description section
with gr.Accordion("📖 Description", open=False, elem_id="description-accordion"):
gr.Markdown(INTRODUCTION_TEXT, elem_classes="markdown-text", elem_id="description-text")
# Get filter options
filter_options = get_filter_options()
# Compact filters in a single horizontal scrollable row
with gr.Row(elem_id="filter-row"):
model_search = gr.Textbox(
label="🔍 Filter by Model",
placeholder="Search...",
value="",
elem_id="model-search",
scale=0
)
category_dropdown = gr.Dropdown(
choices=filter_options["categories"],
value="all",
label="📂 Filter By Category",
allow_custom_value=False,
elem_id="category-filter",
scale=0
)
domain_dropdown = gr.Dropdown(
choices=filter_options["domains"],
value="all",
label="🌐 Filter By Domain",
allow_custom_value=False,
elem_id="domain-filter",
scale=0
)
dataset_dropdown = gr.Dropdown(
choices=filter_options["datasets"],
value="all",
label="📊 Filter by Dataset",
allow_custom_value=False,
elem_id="dataset-filter",
scale=0
)
sort_dropdown = gr.Dropdown(
choices=[
"Rank",
"Model A-Z",
"Organization A-Z",
"Top-Performer ↓",
"Multi-MAPE ↓",
"Uni-MAPE ↓",
"Uni-Multi-MAPE ↑",
"NMAE ↓",
"Date ↑"
],
value="Rank",
label="🔄 Sort",
allow_custom_value=False,
elem_id="sort-filter",
scale=0
)
# Full width table
gr.Markdown("### Models ranked by the number of datasets where they achieve the lowest MAPE (Top-Performer). Click on the model cell to details.")
# Hidden input to handle model selection from table
hidden_model_input = gr.Textbox(visible=False)
# Hidden component to trigger scrolling only for model clicks
scroll_trigger = gr.HTML(visible=False, elem_id="scroll-trigger")
# Main results table with clickable rows
df = create_overall_table()
# Convert DataFrame to list of lists for Gradio
df_values = df.values.tolist()
df_headers = df.columns.tolist()
results_table = gr.Dataframe(
value=df_values,
headers=df_headers,
label="",
interactive=False, # Disable editing but keep select events
elem_id="results-table"
)
refresh_btn = gr.Button("🔄 Refresh Table", variant="primary")
# Model metadata section at bottom
model_inspector_accordion = gr.Accordion("🔍 Model Inspector", open=False, elem_id="model-inspector")
with model_inspector_accordion:
with gr.Row():
with gr.Column(scale=1):
model_selector = gr.Dropdown(
choices=filter_options["models"],
value=None,
label="Select Model",
info="Choose a model to view its metadata",
allow_custom_value=False
)
with gr.Column(scale=3):
metadata_display = gr.Markdown(
value="Select a model to view its metadata.",
label="Model Metadata"
)
# About section
with gr.Accordion("📖 About MUSEval Leaderboard", open=False, elem_id="about-accordion"):
gr.Markdown(BENCHMARKS_TEXT, elem_classes="markdown-text", elem_id="about-text")
# Citation section
with gr.Row():
with gr.Accordion("📙 Citation", open=False, elem_id="citation-accordion"):
citation_button = gr.Textbox(
value=CITATION_BUTTON_TEXT,
label=CITATION_BUTTON_LABEL,
lines=20,
elem_id="citation-button",
show_copy_button=True,
)
# Submit section
with gr.Row():
with gr.Accordion("🚀 Submit Your Model", open=False, elem_id="submit-accordion"):
gr.HTML("""
Submit by creating a pull request with your model's performance here:
🚀 Submit Here
""")
# Event handlers
def update_table(domain, category, dataset, model):
return create_html_table(domain, category, dataset, model)
def clear_filters():
return "all", "all", "all", ""
def reset_other_filters(selected_filter, filter_type):
"""Reset other filters when one is selected"""
if filter_type == "category" and selected_filter != "all":
return gr.update(value="all"), gr.update(value="all") # Reset domain and dataset
elif filter_type == "domain" and selected_filter != "all":
return gr.update(value="all"), gr.update(value="all") # Reset category and dataset
elif filter_type == "dataset" and selected_filter != "all":
return gr.update(value="all"), gr.update(value="all") # Reset category and domain
else:
return gr.update(), gr.update() # No change
def sort_by_dropdown(sort_option, domain, category, dataset, model):
"""Sort table based on dropdown selection - apply filters first, then sort"""
# Map dropdown options to column names
sort_mapping = {
"Rank": "Rank",
"Model A-Z": "Model",
"Organization A-Z": "Organization",
"Top-Performer ↓": "Top-Performer",
"Multi-MAPE ↓": "Multi-MAPE",
"Uni-MAPE ↓": "Uni-MAPE",
"Uni-Multi-MAPE ↑": "Uni-Multi-MAPE",
"NMAE ↓": "NMAE",
"Date ↑": "Submission Date"
}
column_name = sort_mapping.get(sort_option, "Rank")
# First apply filters to get the filtered data
df = create_overall_table(domain_filter=domain, category_filter=category, dataset_filter=dataset, model_filter=model)
# Then sort the filtered data
sorted_df = sort_table_by_column(df, column_name)
# Convert sorted DataFrame back to list format for Gradio Dataframe
return sorted_df.values.tolist()
def update_table_with_sort(sort_option, domain, category, dataset, model):
"""Update table with current filters and sorting"""
return sort_by_dropdown(sort_option, domain, category, dataset, model)
def update_table_with_model_search(model, sort_option, domain, category, dataset):
"""Update table with model search - keep other filters unchanged"""
# Update the table with current filter values (don't reset other filters)
table_result = update_table_with_sort(sort_option, domain, category, dataset, model)
# Return the table and no changes to other dropdowns
return (table_result, gr.update(), gr.update(), gr.update())
def update_table_with_reset(selected_filter, filter_type, sort_option, domain, category, dataset, model):
"""Update table with proper filter reset logic"""
# First, determine what the reset values should be
if filter_type == "category" and selected_filter != "all":
domain = "all"
dataset = "all"
elif filter_type == "domain" and selected_filter != "all":
category = "all"
dataset = "all"
elif filter_type == "dataset" and selected_filter != "all":
category = "all"
domain = "all"
# Update the table with the corrected filter values
table_result = update_table_with_sort(sort_option, domain, category, dataset, model)
# Return the table and the reset updates
reset_updates = reset_other_filters(selected_filter, filter_type)
return (table_result, *reset_updates)
# Connect filters to table updates with mutual exclusivity and sorting
domain_dropdown.change(
fn=lambda domain, category, dataset, model, sort_option: update_table_with_reset(domain, "domain", sort_option, domain, category, dataset, model),
inputs=[domain_dropdown, category_dropdown, dataset_dropdown, model_search, sort_dropdown],
outputs=[results_table, category_dropdown, dataset_dropdown]
)
category_dropdown.change(
fn=lambda domain, category, dataset, model, sort_option: update_table_with_reset(category, "category", sort_option, domain, category, dataset, model),
inputs=[domain_dropdown, category_dropdown, dataset_dropdown, model_search, sort_dropdown],
outputs=[results_table, domain_dropdown, dataset_dropdown]
)
dataset_dropdown.change(
fn=lambda domain, category, dataset, model, sort_option: update_table_with_reset(dataset, "dataset", sort_option, domain, category, dataset, model),
inputs=[domain_dropdown, category_dropdown, dataset_dropdown, model_search, sort_dropdown],
outputs=[results_table, category_dropdown, domain_dropdown]
)
model_search.change(
fn=lambda model, sort_option, domain, category, dataset: update_table_with_model_search(model, sort_option, domain, category, dataset),
inputs=[model_search, sort_dropdown, domain_dropdown, category_dropdown, dataset_dropdown],
outputs=[results_table, domain_dropdown, category_dropdown, dataset_dropdown]
)
refresh_btn.click(
fn=update_table_with_sort,
inputs=[domain_dropdown, category_dropdown, dataset_dropdown, model_search, sort_dropdown],
outputs=results_table
)
# Sort dropdown event handler - independent of filters
sort_dropdown.change(
fn=sort_by_dropdown,
inputs=[sort_dropdown, domain_dropdown, category_dropdown, dataset_dropdown, model_search],
outputs=results_table
)
# Model selector event handler
model_selector.change(
fn=create_model_metadata_display,
inputs=[model_selector],
outputs=[metadata_display]
)
# Model column cell selection handler (with controlled scrolling)
def handle_model_column_clicks(evt: gr.SelectData):
"""Handle only model column cell clicks for model selection"""
print(f"DEBUG: Click detected - Row: {evt.index[0]}, Column: {evt.index[1]}, Value: {evt.value}")
row_idx = evt.index[0]
col_idx = evt.index[1]
# Only handle model column clicks (column 0)
if col_idx == 0: # Model column
print("🎯 MODEL COLUMN CELL CLICK DETECTED!")
if hasattr(evt, 'row_value') and evt.row_value is not None and len(evt.row_value) > 0:
model_name = evt.row_value[0]
print(f"🎯 Model selected: {model_name}")
# Return model selection, accordion expansion, and scroll trigger
return gr.update(value=model_name), gr.update(open=True), gr.update(value="scroll")
elif evt.value is not None:
model_name = evt.value
print(f"🎯 Model selected: {model_name}")
# Return model selection, accordion expansion, and scroll trigger
return gr.update(value=model_name), gr.update(open=True), gr.update(value="scroll")
else:
print("🎯 OTHER COLUMN CELL CLICK - NO ACTION (NO SCROLLING)")
# For non-model column clicks, return no changes (no scrolling)
return gr.update(), gr.update(), gr.update()
results_table.select(
fn=handle_model_column_clicks,
inputs=[],
outputs=[model_selector, model_inspector_accordion, scroll_trigger]
)
# Handle scroll trigger - only scroll when model is selected
def handle_scroll_trigger(trigger_value):
"""Handle scrolling only when model is selected"""
if trigger_value == "scroll":
print("🎯 SCROLL TRIGGER ACTIVATED!")
# Trigger scrolling by updating the scroll trigger and ensuring accordion is open
return gr.update(value="scrolled"), gr.update(open=True)
return gr.update(), gr.update()
scroll_trigger.change(
fn=handle_scroll_trigger,
inputs=[scroll_trigger],
outputs=[scroll_trigger, model_inspector_accordion],
scroll_to_output=True
)
# Handle change events from interactive table
def handle_table_changes(new_value):
"""Handle changes from interactive table behavior"""
print("=" * 50)
print("DEBUG: Table Change Event Detected")
print("=" * 50)
print(f"New value: {new_value}")
print(f"New value type: {type(new_value)}")
print("=" * 50)
return gr.update()
results_table.change(
fn=handle_table_changes,
inputs=[results_table],
outputs=[]
)
# Hidden input event handler - when model is selected from table
def update_model_from_hidden(hidden_value):
if hidden_value:
return gr.update(value=hidden_value)
return gr.update()
hidden_model_input.change(
fn=update_model_from_hidden,
inputs=[hidden_model_input],
outputs=[model_selector]
)
return demo
# Start scheduler if available
if SCHEDULER_AVAILABLE:
scheduler = BackgroundScheduler()
scheduler.start()
else:
scheduler = None
# Launch the demo
if __name__ == "__main__":
demo = create_leaderboard_interface()
demo.queue(default_concurrency_limit=40).launch()