Merge branch 'feat-add-versions-to-benchmarks-1015' into pr/28

app.py

@@ -1,105 +1,63 @@
 import gradio as gr
+import pandas as pd
 from apscheduler.schedulers.background import BackgroundScheduler
-from huggingface_hub import snapshot_download
 
 from src.about import (
     INTRODUCTION_TEXT,
-    BENCHMARKS_TEXT,
-    TITLE,
-    EVALUATION_QUEUE_TEXT
+    TITLE
 )
 from src.benchmarks import (
-    DOMAIN_COLS_QA,
-    LANG_COLS_QA,
-    DOMAIN_COLS_LONG_DOC,
-    LANG_COLS_LONG_DOC,
-    METRIC_LIST,
-    DEFAULT_METRIC_QA,
-    DEFAULT_METRIC_LONG_DOC
+    QABenchmarks,
+    LongDocBenchmarks
 )
 from src.display.css_html_js import custom_css
-from src.display.utils import (
-    COL_NAME_IS_ANONYMOUS,
-    COL_NAME_REVISION,
-    COL_NAME_TIMESTAMP,
-    COL_NAME_RERANKING_MODEL,
-    COL_NAME_RETRIEVAL_MODEL
-)
 from src.envs import (
     API,
     EVAL_RESULTS_PATH,
-    REPO_ID,
-    RESULTS_REPO,
-    TOKEN,
-    BM25_LINK,
-    BENCHMARK_VERSION_LIST,
-    LATEST_BENCHMARK_VERSION
+    REPO_ID, DEFAULT_METRIC_QA, DEFAULT_METRIC_LONG_DOC, METRIC_LIST, LATEST_BENCHMARK_VERSION, COL_NAME_RERANKING_MODEL, COL_NAME_RETRIEVAL_MODEL, BM25_LINK, BENCHMARK_VERSION_LIST
 )
-from src.read_evals import (
-    get_raw_eval_results,
-    get_leaderboard_df
+from src.loaders import (
+    load_eval_results
 )
 from src.utils import (
     update_metric,
-    upload_file,
-    get_default_cols,
-    submit_results,
+    set_listeners,
     reset_rank,
-    remove_html
+    remove_html, upload_file, submit_results
 )
 from src.display.gradio_formatting import (
     get_version_dropdown,
     get_search_bar,
     get_reranking_dropdown,
+    get_noreranking_dropdown,
     get_metric_dropdown,
     get_domain_dropdown,
     get_language_dropdown,
     get_anonymous_checkbox,
     get_revision_and_ts_checkbox,
-    get_leaderboard_table,
-    get_noreranking_dropdown
+    get_leaderboard_table
 )
-from src.display.gradio_listener import set_listeners
-
-def restart_space():
-    API.restart_space(repo_id=REPO_ID)
 
+from src.about import EVALUATION_QUEUE_TEXT, BENCHMARKS_TEXT
 
-try:
-    snapshot_download(
-        repo_id=RESULTS_REPO, local_dir=EVAL_RESULTS_PATH, repo_type="dataset", tqdm_class=None, etag_timeout=30,
-        token=TOKEN
-    )
-except Exception as e:
-    print(f'failed to download')
-    restart_space()
 
-raw_data = get_raw_eval_results(f"{EVAL_RESULTS_PATH}/{LATEST_BENCHMARK_VERSION}")
-
-original_df_qa = get_leaderboard_df(
-    raw_data, task='qa', metric=DEFAULT_METRIC_QA)
-original_df_long_doc = get_leaderboard_df(
-    raw_data, task='long-doc', metric=DEFAULT_METRIC_LONG_DOC)
-print(f'raw data: {len(raw_data)}')
-print(f'QA data loaded: {original_df_qa.shape}')
-print(f'Long-Doc data loaded: {len(original_df_long_doc)}')
-
-leaderboard_df_qa = original_df_qa.copy()
-# leaderboard_df_qa = leaderboard_df_qa[has_no_nan_values(df, _benchmark_cols)]
-shown_columns_qa, types_qa = get_default_cols(
-    'qa', leaderboard_df_qa.columns, add_fix_cols=True)
-leaderboard_df_qa = leaderboard_df_qa[~leaderboard_df_qa[COL_NAME_IS_ANONYMOUS]][shown_columns_qa]
-leaderboard_df_qa.drop([COL_NAME_REVISION, COL_NAME_TIMESTAMP], axis=1, inplace=True)
+def restart_space():
+    API.restart_space(repo_id=REPO_ID)
 
-leaderboard_df_long_doc = original_df_long_doc.copy()
-shown_columns_long_doc, types_long_doc = get_default_cols(
-    'long-doc', leaderboard_df_long_doc.columns, add_fix_cols=True)
-leaderboard_df_long_doc = leaderboard_df_long_doc[~leaderboard_df_long_doc[COL_NAME_IS_ANONYMOUS]][shown_columns_long_doc]
-leaderboard_df_long_doc.drop([COL_NAME_REVISION, COL_NAME_TIMESTAMP], axis=1, inplace=True)
 
-# select reranking model
-reranking_models = sorted(list(frozenset([eval_result.reranking_model for eval_result in raw_data])))
+# try:
+#     snapshot_download(
+#         repo_id=RESULTS_REPO, local_dir=EVAL_RESULTS_PATH, repo_type="dataset", tqdm_class=None, etag_timeout=30,
+#         token=TOKEN
+#     )
+# except Exception as e:
+#     print(f'failed to download')
+#     restart_space()
 
+global data
+data = load_eval_results(EVAL_RESULTS_PATH)
+global datastore
+datastore = data[LATEST_BENCHMARK_VERSION]
 
 def update_metric_qa(
         metric: str,
@@ -108,9 +66,10 @@ def update_metric_qa(
         reranking_model: list,
         query: str,
         show_anonymous: bool,
-        show_revision_and_timestamp,
+        show_revision_and_timestamp: bool,
 ):
-    return update_metric(raw_data, 'qa', metric, domains, langs, reranking_model, query, show_anonymous, show_revision_and_timestamp)
+    return update_metric(datastore, 'qa', metric, domains, langs, reranking_model, query, show_anonymous, show_revision_and_timestamp)
+
 
 def update_metric_long_doc(
         metric: str,
@@ -121,7 +80,37 @@ def update_metric_long_doc(
         show_anonymous: bool,
         show_revision_and_timestamp,
 ):
-    return update_metric(raw_data, "long-doc", metric, domains, langs, reranking_model, query, show_anonymous, show_revision_and_timestamp)
+    return update_metric(datastore, "long-doc", metric, domains, langs, reranking_model, query, show_anonymous, show_revision_and_timestamp)
+
+
+def update_datastore(version):
+    print("triggered update_datastore")
+    global datastore
+    global data
+    datastore = data[version]
+    selected_domains = get_domain_dropdown(QABenchmarks[datastore.slug])
+    selected_langs = get_language_dropdown(QABenchmarks[datastore.slug])
+    selected_rerankings = get_reranking_dropdown(datastore.reranking_models)
+    leaderboard_table = get_leaderboard_table(
+        datastore.leaderboard_df_qa, datastore.types_qa)
+    hidden_leaderboard_table = get_leaderboard_table(
+        datastore.raw_df_qa, datastore.types_qa, visible=False)
+    return selected_domains, selected_langs, selected_rerankings, leaderboard_table, hidden_leaderboard_table
+
+
+def update_datastore_long_doc(version):
+    global datastore
+    global data
+    print("triggered update_datastore_long_doc")
+    datastore = data[version]
+    selected_domains = get_domain_dropdown(LongDocBenchmarks[datastore.slug])
+    selected_langs = get_language_dropdown(LongDocBenchmarks[datastore.slug])
+    selected_rerankings = get_reranking_dropdown(datastore.reranking_models)
+    leaderboard_table = get_leaderboard_table(
+        datastore.leaderboard_df_long_doc, datastore.types_long_doc)
+    hidden_leaderboard_table = get_leaderboard_table(
+        datastore.raw_df_long_doc, datastore.types_long_doc, visible=False)
+    return selected_domains, selected_langs, selected_rerankings, leaderboard_table, hidden_leaderboard_table
 
 
 demo = gr.Blocks(css=custom_css)
@@ -140,11 +129,10 @@ with demo:
                     with gr.Column(min_width=320):
                         # select domain
                         with gr.Row():
-                            selected_domains = get_domain_dropdown(DOMAIN_COLS_QA, DOMAIN_COLS_QA)
+                            selected_domains = get_domain_dropdown(QABenchmarks[datastore.slug])
                         # select language
                         with gr.Row():
-                            selected_langs = get_language_dropdown(LANG_COLS_QA, LANG_COLS_QA)
-
+                            selected_langs = get_language_dropdown(QABenchmarks[datastore.slug])
                     with gr.Column():
                         # select the metric
                         selected_metric = get_metric_dropdown(METRIC_LIST, DEFAULT_METRIC_QA)
@@ -160,16 +148,26 @@ with demo:
                                 search_bar = get_search_bar()
                             # select reranking models
                             with gr.Column():
-                                selected_rerankings = get_reranking_dropdown(reranking_models)
-                        leaderboard_table = get_leaderboard_table(leaderboard_df_qa, types_qa)
+                                selected_rerankings = get_reranking_dropdown(datastore.reranking_models)
+                        #  shown_table
+                        lb_table = get_leaderboard_table(
+                            datastore.leaderboard_df_qa, datastore.types_qa)
                         # Dummy leaderboard for handling the case when the user uses backspace key
-                        hidden_leaderboard_table_for_search = get_leaderboard_table(original_df_qa, types_qa, visible=False)
+                        hidden_lb_table = get_leaderboard_table(
+                            datastore.raw_df_qa, datastore.types_qa, visible=False)
+
+                        selected_version.change(
+                            update_datastore,
+                            [selected_version,],
+                            [selected_domains, selected_langs, selected_rerankings, lb_table, hidden_lb_table]
+                        )
 
                         set_listeners(
                             "qa",
-                            leaderboard_table,
-                            hidden_leaderboard_table_for_search,
+                            lb_table,
+                            hidden_lb_table,
                             search_bar,
+                            selected_version,
                             selected_domains,
                             selected_langs,
                             selected_rerankings,
@@ -189,28 +187,45 @@ with demo:
                                 show_anonymous,
                                 show_revision_and_timestamp,
                             ],
-                            leaderboard_table,
+                            lb_table,
                             queue=True
                         )
+
                     with gr.TabItem("Retrieval Only", id=11):
                         with gr.Row():
                             with gr.Column(scale=1):
                                 search_bar_retriever = get_search_bar()
                             with gr.Column(scale=1):
                                 selected_noreranker = get_noreranking_dropdown()
-                        lb_df_retriever = leaderboard_df_qa[leaderboard_df_qa[COL_NAME_RERANKING_MODEL] == "NoReranker"]
+
+                        lb_df_retriever = datastore.leaderboard_df_qa[datastore.leaderboard_df_qa[COL_NAME_RERANKING_MODEL] == "NoReranker"]
                         lb_df_retriever = reset_rank(lb_df_retriever)
-                        lb_table_retriever = get_leaderboard_table(lb_df_retriever, types_qa)
+                        lb_table_retriever = get_leaderboard_table(
+                            lb_df_retriever, datastore.types_qa)
+
                         # Dummy leaderboard for handling the case when the user uses backspace key
-                        hidden_lb_df_retriever = original_df_qa[original_df_qa[COL_NAME_RERANKING_MODEL] == "NoReranker"]
+                        hidden_lb_df_retriever = datastore.raw_df_qa[datastore.raw_df_qa[COL_NAME_RERANKING_MODEL] == "NoReranker"]
                         hidden_lb_df_retriever = reset_rank(hidden_lb_df_retriever)
-                        hidden_lb_table_retriever = get_leaderboard_table(hidden_lb_df_retriever, types_qa, visible=False)
+                        hidden_lb_table_retriever = get_leaderboard_table(hidden_lb_df_retriever, datastore.types_qa, visible=False)
+
+                        selected_version.change(
+                            update_datastore,
+                            [selected_version,],
+                            [
+                                selected_domains,
+                                selected_langs,
+                                selected_noreranker,
+                                lb_table_retriever,
+                                hidden_lb_table_retriever
+                            ]
+                        )
 
                         set_listeners(
                             "qa",
                             lb_table_retriever,
                             hidden_lb_table_retriever,
                             search_bar_retriever,
+                            selected_version,
                             selected_domains,
                             selected_langs,
                             selected_noreranker,
@@ -234,7 +249,12 @@ with demo:
                             queue=True
                         )
                     with gr.TabItem("Reranking Only", id=12):
-                        lb_df_reranker = leaderboard_df_qa[leaderboard_df_qa[COL_NAME_RETRIEVAL_MODEL] == BM25_LINK]
+                        lb_df_reranker = \
+                            datastore.leaderboard_df_qa[
+                                datastore.leaderboard_df_qa[
+                                    COL_NAME_RETRIEVAL_MODEL
+                                ] == BM25_LINK
+                            ]
                         lb_df_reranker = reset_rank(lb_df_reranker)
                         reranking_models_reranker = lb_df_reranker[COL_NAME_RERANKING_MODEL].apply(remove_html).unique().tolist()
                         with gr.Row():
@@ -242,11 +262,26 @@ with demo:
                                 selected_rerankings_reranker = get_reranking_dropdown(reranking_models_reranker)
                             with gr.Column(scale=1):
                                 search_bar_reranker = gr.Textbox(show_label=False, visible=False)
-                        lb_table_reranker = get_leaderboard_table(lb_df_reranker, types_qa)
-                        hidden_lb_df_reranker = original_df_qa[original_df_qa[COL_NAME_RETRIEVAL_MODEL] == BM25_LINK]
+                        lb_table_reranker = get_leaderboard_table(
+                            lb_df_reranker, datastore.types_qa)
+
+                        hidden_lb_df_reranker = datastore.raw_df_qa[datastore.raw_df_qa[COL_NAME_RETRIEVAL_MODEL] == BM25_LINK]
                         hidden_lb_df_reranker = reset_rank(hidden_lb_df_reranker)
                         hidden_lb_table_reranker = get_leaderboard_table(
-                            hidden_lb_df_reranker, types_qa, visible=False
+                            hidden_lb_df_reranker,
+                            datastore.types_qa, visible=False
+                        )
+
+                        selected_version.change(
+                            update_datastore,
+                            [selected_version,],
+                            [
+                                selected_domains,
+                                selected_langs,
+                                selected_rerankings_reranker,
+                                lb_table_reranker,
+                                hidden_lb_table_reranker
+                            ]
                         )
 
                         set_listeners(
@@ -254,6 +289,7 @@ with demo:
                             lb_table_reranker,
                             hidden_lb_table_reranker,
                             search_bar_reranker,
+                            selected_version,
                             selected_domains,
                             selected_langs,
                             selected_rerankings_reranker,
@@ -280,12 +316,10 @@ with demo:
                     with gr.Column(min_width=320):
                         # select domain
                         with gr.Row():
-                            selected_domains = get_domain_dropdown(DOMAIN_COLS_LONG_DOC, DOMAIN_COLS_LONG_DOC)
+                            selected_domains = get_domain_dropdown(LongDocBenchmarks[datastore.slug])
                         # select language
                         with gr.Row():
-                            selected_langs = get_language_dropdown(
-                                LANG_COLS_LONG_DOC, LANG_COLS_LONG_DOC
-                            )
+                            selected_langs = get_language_dropdown(LongDocBenchmarks[datastore.slug])
                     with gr.Column():
                         # select the metric
                         with gr.Row():
@@ -301,22 +335,35 @@ with demo:
                                 search_bar = get_search_bar()
                             # select reranking model
                             with gr.Column():
-                                selected_rerankings = get_reranking_dropdown(reranking_models)
+                                selected_rerankings = get_reranking_dropdown(datastore.reranking_models)
 
-                        lb_table = get_leaderboard_table(
-                            leaderboard_df_long_doc, types_long_doc
+                        lb_table_long_doc = get_leaderboard_table(
+                            datastore.leaderboard_df_long_doc, datastore.types_long_doc
                         )
 
                         # Dummy leaderboard for handling the case when the user uses backspace key
-                        hidden_lb_table_for_search = get_leaderboard_table(
-                            original_df_long_doc, types_long_doc, visible=False
+                        hidden_lb_table_long_doc = get_leaderboard_table(
+                            datastore.raw_df_long_doc, datastore.types_long_doc, visible=False
+                        )
+
+                        selected_version.change(
+                            update_datastore_long_doc,
+                            [selected_version,],
+                            [
+                                selected_domains,
+                                selected_langs,
+                                selected_rerankings,
+                                lb_table_long_doc,
+                                hidden_lb_table_long_doc
+                            ]
                         )
 
                         set_listeners(
                             "long-doc",
-                            lb_table,
-                            hidden_lb_table_for_search,
+                            lb_table_long_doc,
+                            hidden_lb_table_long_doc,
                             search_bar,
+                            selected_version,
                             selected_domains,
                             selected_langs,
                             selected_rerankings,
@@ -336,7 +383,7 @@ with demo:
                                 show_anonymous,
                                 show_revision_and_timestamp
                             ],
-                            lb_table,
+                            lb_table_long_doc,
                             queue=True
                         )
                     with gr.TabItem("Retrieval Only", id=21):
@@ -345,18 +392,31 @@ with demo:
                                 search_bar_retriever = get_search_bar()
                             with gr.Column(scale=1):
                                 selected_noreranker = get_noreranking_dropdown()
-                        lb_df_retriever_long_doc = leaderboard_df_long_doc[
-                            leaderboard_df_long_doc[COL_NAME_RERANKING_MODEL] == "NoReranker"
+                        lb_df_retriever_long_doc = datastore.leaderboard_df_long_doc[
+                            datastore.leaderboard_df_long_doc[COL_NAME_RERANKING_MODEL] == "NoReranker"
                         ]
                         lb_df_retriever_long_doc = reset_rank(lb_df_retriever_long_doc)
-                        hidden_lb_db_retriever_long_doc = original_df_long_doc[
-                            original_df_long_doc[COL_NAME_RERANKING_MODEL] == "NoReranker"
-                        ]
-                        hidden_lb_db_retriever_long_doc = reset_rank(hidden_lb_db_retriever_long_doc)
                         lb_table_retriever_long_doc = get_leaderboard_table(
-                            lb_df_retriever_long_doc, types_long_doc)
+                            lb_df_retriever_long_doc, datastore.types_long_doc)
+
+                        hidden_lb_df_retriever_long_doc = datastore.raw_df_long_doc[
+                            datastore.raw_df_long_doc[COL_NAME_RERANKING_MODEL] == "NoReranker"
+                        ]
+                        hidden_lb_df_retriever_long_doc = reset_rank(hidden_lb_df_retriever_long_doc)
                         hidden_lb_table_retriever_long_doc = get_leaderboard_table(
-                            hidden_lb_db_retriever_long_doc, types_long_doc, visible=False
+                            hidden_lb_df_retriever_long_doc, datastore.types_long_doc, visible=False
+                        )
+
+                        selected_version.change(
+                            update_datastore_long_doc,
+                            [selected_version,],
+                            [
+                                selected_domains,
+                                selected_langs,
+                                selected_noreranker,
+                                lb_table_retriever_long_doc,
+                                hidden_lb_table_retriever_long_doc
+                            ]
                         )
 
                         set_listeners(
@@ -364,6 +424,7 @@ with demo:
                             lb_table_retriever_long_doc,
                             hidden_lb_table_retriever_long_doc,
                             search_bar_retriever,
+                            selected_version,
                             selected_domains,
                             selected_langs,
                             selected_noreranker,
@@ -386,8 +447,11 @@ with demo:
                             queue=True
                         )
                     with gr.TabItem("Reranking Only", id=22):
-                        lb_df_reranker_ldoc = leaderboard_df_long_doc[
-                            leaderboard_df_long_doc[COL_NAME_RETRIEVAL_MODEL] == BM25_LINK
+                        lb_df_reranker_ldoc = \
+                            datastore.leaderboard_df_long_doc[
+                                datastore.leaderboard_df_long_doc[
+                                    COL_NAME_RETRIEVAL_MODEL
+                                ] == BM25_LINK
                             ]
                         lb_df_reranker_ldoc = reset_rank(lb_df_reranker_ldoc)
                         reranking_models_reranker_ldoc = lb_df_reranker_ldoc[COL_NAME_RERANKING_MODEL].apply(remove_html).unique().tolist()
@@ -396,11 +460,23 @@ with demo:
                                 selected_rerankings_reranker_ldoc = get_reranking_dropdown(reranking_models_reranker_ldoc)
                             with gr.Column(scale=1):
                                 search_bar_reranker_ldoc = gr.Textbox(show_label=False, visible=False)
-                        lb_table_reranker_ldoc = get_leaderboard_table(lb_df_reranker_ldoc, types_long_doc)
-                        hidden_lb_df_reranker_ldoc = original_df_long_doc[original_df_long_doc[COL_NAME_RETRIEVAL_MODEL] == BM25_LINK]
+                        lb_table_reranker_ldoc = get_leaderboard_table(lb_df_reranker_ldoc, datastore.types_long_doc)
+                        hidden_lb_df_reranker_ldoc = datastore.raw_df_long_doc[datastore.raw_df_long_doc[COL_NAME_RETRIEVAL_MODEL] == BM25_LINK]
                         hidden_lb_df_reranker_ldoc = reset_rank(hidden_lb_df_reranker_ldoc)
                         hidden_lb_table_reranker_ldoc = get_leaderboard_table(
-                            hidden_lb_df_reranker_ldoc, types_long_doc, visible=False
+                            hidden_lb_df_reranker_ldoc, datastore.types_long_doc, visible=False
+                        )
+
+                        selected_version.change(
+                            update_datastore_long_doc,
+                            [selected_version,],
+                            [
+                                selected_domains,
+                                selected_langs,
+                                selected_rerankings_reranker_ldoc,
+                                lb_table_reranker_ldoc,
+                                hidden_lb_table_reranker_ldoc
+                            ]
                         )
 
                         set_listeners(
@@ -408,6 +484,7 @@ with demo:
                             lb_table_reranker_ldoc,
                             hidden_lb_table_reranker_ldoc,
                             search_bar_reranker_ldoc,
+                            selected_version,
                             selected_domains,
                             selected_langs,
                             selected_rerankings_reranker_ldoc,
@@ -503,3 +580,5 @@ if __name__ == "__main__":
     scheduler.start()
     demo.queue(default_concurrency_limit=40)
     demo.launch()
+
+

src/benchmarks.py

@@ -1,7 +1,10 @@
 from dataclasses import dataclass
 from enum import Enum
+
 from air_benchmark.tasks.tasks import BenchmarkTable
 
+from src.envs import METRIC_LIST
+
 
 def get_safe_name(name: str):
     """Get RFC 1123 compatible safe name"""
@@ -12,40 +15,6 @@ def get_safe_name(name: str):
         if (character.isalnum() or character == '_'))
 
 
-METRIC_LIST = [
-    "ndcg_at_1",
-    "ndcg_at_3",
-    "ndcg_at_5",
-    "ndcg_at_10",
-    "ndcg_at_100",
-    "ndcg_at_1000",
-    "map_at_1",
-    "map_at_3",
-    "map_at_5",
-    "map_at_10",
-    "map_at_100",
-    "map_at_1000",
-    "recall_at_1",
-    "recall_at_3",
-    "recall_at_5",
-    "recall_at_10",
-    "recall_at_100",
-    "recall_at_1000",
-    "precision_at_1",
-    "precision_at_3",
-    "precision_at_5",
-    "precision_at_10",
-    "precision_at_100",
-    "precision_at_1000",
-    "mrr_at_1",
-    "mrr_at_3",
-    "mrr_at_5",
-    "mrr_at_10",
-    "mrr_at_100",
-    "mrr_at_1000"
-]
-
-
 @dataclass
 class Benchmark:
     name: str  # [domain]_[language]_[metric], task_key in the json file,
@@ -56,37 +25,53 @@ class Benchmark:
     task: str
 
 
-qa_benchmark_dict = {}
-long_doc_benchmark_dict = {}
-for task, domain_dict in BenchmarkTable['AIR-Bench_24.04'].items():
-    for domain, lang_dict in domain_dict.items():
-        for lang, dataset_list in lang_dict.items():
-            if task == "qa":
-                benchmark_name = f"{domain}_{lang}"
-                benchmark_name = get_safe_name(benchmark_name)
-                col_name = benchmark_name
-                for metric in dataset_list:
-                    qa_benchmark_dict[benchmark_name] = Benchmark(benchmark_name, metric, col_name, domain, lang, task)
-            elif task == "long-doc":
-                for dataset in dataset_list:
-                    benchmark_name = f"{domain}_{lang}_{dataset}"
-                    benchmark_name = get_safe_name(benchmark_name)
+# create a function return an enum class containing all the benchmarks
+def get_benchmarks_enum(benchmark_version, task_type):
+    benchmark_dict = {}
+    if task_type == "qa":
+        for task, domain_dict in BenchmarkTable[benchmark_version].items():
+            if task != task_type:
+                continue
+            for domain, lang_dict in domain_dict.items():
+                for lang, dataset_list in lang_dict.items():
+                    benchmark_name = get_safe_name(f"{domain}_{lang}")
                     col_name = benchmark_name
-                    for metric in METRIC_LIST:
-                        long_doc_benchmark_dict[benchmark_name] = Benchmark(benchmark_name, metric, col_name, domain,
-                                                                            lang, task)
+                    for metric in dataset_list:
+                        if "test" not in dataset_list[metric]["splits"]:
+                            continue
+                        benchmark_dict[benchmark_name] = \
+                            Benchmark(benchmark_name, metric, col_name, domain, lang, task)
+    elif task_type == "long-doc":
+        for task, domain_dict in BenchmarkTable[benchmark_version].items():
+            if task != task_type:
+                continue
+            for domain, lang_dict in domain_dict.items():
+                for lang, dataset_list in lang_dict.items():
+                    for dataset in dataset_list:
+                        benchmark_name = f"{domain}_{lang}_{dataset}"
+                        benchmark_name = get_safe_name(benchmark_name)
+                        col_name = benchmark_name
+                        if "test" not in dataset_list[dataset]["splits"]:
+                            continue
+                        for metric in METRIC_LIST:
+                            benchmark_dict[benchmark_name] = \
+                                Benchmark(benchmark_name, metric, col_name, domain, lang, task)
+    return benchmark_dict
 
-BenchmarksQA = Enum('BenchmarksQA', qa_benchmark_dict)
-BenchmarksLongDoc = Enum('BenchmarksLongDoc', long_doc_benchmark_dict)
 
-BENCHMARK_COLS_QA = [c.col_name for c in qa_benchmark_dict.values()]
-BENCHMARK_COLS_LONG_DOC = [c.col_name for c in long_doc_benchmark_dict.values()]
+versions = ("AIR-Bench_24.04", "AIR-Bench_24.05")
+qa_benchmark_dict = {}
+for version in versions:
+    safe_version_name = get_safe_name(version)[-4:]
+    qa_benchmark_dict[safe_version_name] = Enum(f"QABenchmarks_{safe_version_name}", get_benchmarks_enum(version, "qa"))
 
-DOMAIN_COLS_QA = list(frozenset([c.domain for c in qa_benchmark_dict.values()]))
-LANG_COLS_QA = list(frozenset([c.lang for c in qa_benchmark_dict.values()]))
+long_doc_benchmark_dict = {}
+for version in versions:
+    safe_version_name = get_safe_name(version)[-4:]
+    long_doc_benchmark_dict[safe_version_name] = Enum(f"LongDocBenchmarks_{safe_version_name}", get_benchmarks_enum(version, "long-doc"))
 
-DOMAIN_COLS_LONG_DOC = list(frozenset([c.domain for c in long_doc_benchmark_dict.values()]))
-LANG_COLS_LONG_DOC = list(frozenset([c.lang for c in long_doc_benchmark_dict.values()]))
+# _qa_benchmark_dict, = get_benchmarks_enum('AIR-Bench_24.04', "qa")
+# _long_doc_benchmark_dict = get_benchmarks_enum('AIR-Bench_24.04', "long-doc")
 
-DEFAULT_METRIC_QA = "ndcg_at_10"
-DEFAULT_METRIC_LONG_DOC = "recall_at_10"
+QABenchmarks = Enum('QABenchmarks', qa_benchmark_dict)
+LongDocBenchmarks = Enum('LongDocBenchmarks', long_doc_benchmark_dict)

src/display/{utils.py → columns.py}

@@ -1,6 +1,8 @@
 from dataclasses import dataclass, make_dataclass
 
-from src.benchmarks import BenchmarksQA, BenchmarksLongDoc
+from src.benchmarks import QABenchmarks, LongDocBenchmarks
+from src.envs import COL_NAME_AVG, COL_NAME_RETRIEVAL_MODEL, COL_NAME_RERANKING_MODEL, COL_NAME_RETRIEVAL_MODEL_LINK, \
+    COL_NAME_RERANKING_MODEL_LINK, COL_NAME_RANK, COL_NAME_REVISION, COL_NAME_TIMESTAMP, COL_NAME_IS_ANONYMOUS
 
 
 def fields(raw_class):
@@ -19,17 +21,6 @@ class ColumnContent:
     never_hidden: bool = False
 
 
-COL_NAME_AVG = "Average ⬆️"
-COL_NAME_RETRIEVAL_MODEL = "Retrieval Method"
-COL_NAME_RERANKING_MODEL = "Reranking Model"
-COL_NAME_RETRIEVAL_MODEL_LINK = "Retrieval Model LINK"
-COL_NAME_RERANKING_MODEL_LINK = "Reranking Model LINK"
-COL_NAME_RANK = "Rank 🏆"
-COL_NAME_REVISION = "Revision"
-COL_NAME_TIMESTAMP = "Submission Date"
-COL_NAME_IS_ANONYMOUS = "Anonymous Submission"
-
-
 def get_default_auto_eval_column_dict():
     auto_eval_column_dict = []
     # Init
@@ -37,10 +28,12 @@ def get_default_auto_eval_column_dict():
         ["rank", ColumnContent, ColumnContent(COL_NAME_RANK, "number", True)]
     )
     auto_eval_column_dict.append(
-        ["retrieval_model", ColumnContent, ColumnContent(COL_NAME_RETRIEVAL_MODEL, "markdown", True, hidden=False, never_hidden=True)]
+        ["retrieval_model", ColumnContent,
+         ColumnContent(COL_NAME_RETRIEVAL_MODEL, "markdown", True, hidden=False, never_hidden=True)]
     )
     auto_eval_column_dict.append(
-        ["reranking_model", ColumnContent, ColumnContent(COL_NAME_RERANKING_MODEL, "markdown", True, hidden=False, never_hidden=True)]
+        ["reranking_model", ColumnContent,
+         ColumnContent(COL_NAME_RERANKING_MODEL, "markdown", True, hidden=False, never_hidden=True)]
     )
     auto_eval_column_dict.append(
         ["revision", ColumnContent, ColumnContent(COL_NAME_REVISION, "markdown", True, never_hidden=True)]
@@ -52,10 +45,12 @@ def get_default_auto_eval_column_dict():
         ["average", ColumnContent, ColumnContent(COL_NAME_AVG, "number", True)]
     )
     auto_eval_column_dict.append(
-        ["retrieval_model_link", ColumnContent, ColumnContent(COL_NAME_RETRIEVAL_MODEL_LINK, "markdown", False, hidden=True, never_hidden=False)]
+        ["retrieval_model_link", ColumnContent,
+         ColumnContent(COL_NAME_RETRIEVAL_MODEL_LINK, "markdown", False, hidden=True, never_hidden=False)]
     )
     auto_eval_column_dict.append(
-        ["reranking_model_link", ColumnContent, ColumnContent(COL_NAME_RERANKING_MODEL_LINK, "markdown", False, hidden=True, never_hidden=False)]
+        ["reranking_model_link", ColumnContent,
+         ColumnContent(COL_NAME_RERANKING_MODEL_LINK, "markdown", False, hidden=True, never_hidden=False)]
     )
     auto_eval_column_dict.append(
         ["is_anonymous", ColumnContent, ColumnContent(COL_NAME_IS_ANONYMOUS, "bool", False, hidden=True)]
@@ -63,10 +58,10 @@ def get_default_auto_eval_column_dict():
     return auto_eval_column_dict
 
 
-def make_autoevalcolumn(cls_name="BenchmarksQA", benchmarks=BenchmarksQA):
+def make_autoevalcolumn(cls_name, benchmarks):
     auto_eval_column_dict = get_default_auto_eval_column_dict()
-    ## Leaderboard columns
-    for benchmark in benchmarks:
+    # Leaderboard columns
+    for benchmark in list(benchmarks.value):
         auto_eval_column_dict.append(
             [benchmark.name, ColumnContent, ColumnContent(benchmark.value.col_name, "number", True)]
         )
@@ -75,19 +70,28 @@ def make_autoevalcolumn(cls_name="BenchmarksQA", benchmarks=BenchmarksQA):
     return make_dataclass(cls_name, auto_eval_column_dict, frozen=True)
 
 
-AutoEvalColumnQA = make_autoevalcolumn(
-    "AutoEvalColumnQA", BenchmarksQA)
-AutoEvalColumnLongDoc = make_autoevalcolumn(
-    "AutoEvalColumnLongDoc", BenchmarksLongDoc)
+def get_default_col_names_and_types(benchmarks):
+    AutoEvalColumn = make_autoevalcolumn("AutoEvalColumn", benchmarks)
+    col_names = [c.name for c in fields(AutoEvalColumn) if not c.hidden]
+    col_types = [c.type for c in fields(AutoEvalColumn) if not c.hidden]
+    return col_names, col_types
 
+# AutoEvalColumnQA = make_autoevalcolumn("AutoEvalColumnQA", QABenchmarks)
+# COLS_QA = [c.name for c in fields(AutoEvalColumnQA) if not c.hidden]
+# TYPES_QA = [c.type for c in fields(AutoEvalColumnQA) if not c.hidden]
 
-# Column selection
-COLS_QA = [c.name for c in fields(AutoEvalColumnQA) if not c.hidden]
-COLS_LONG_DOC = [c.name for c in fields(AutoEvalColumnLongDoc) if not c.hidden]
-TYPES_QA = [c.type for c in fields(AutoEvalColumnQA) if not c.hidden]
-TYPES_LONG_DOC = [c.type for c in fields(AutoEvalColumnLongDoc) if not c.hidden]
-COLS_LITE = [c.name for c in fields(AutoEvalColumnQA) if c.displayed_by_default and not c.hidden]
 
-QA_BENCHMARK_COLS = [t.value.col_name for t in BenchmarksQA]
+def get_fixed_col_names_and_types():
+    fixed_cols = get_default_auto_eval_column_dict()[:-3]
+    return [c.name for _, _, c in fixed_cols], [c.type for _, _, c in fixed_cols]
+
+# fixed_cols = get_default_auto_eval_column_dict()[:-3]
+# FIXED_COLS = [c.name for _, _, c in fixed_cols]
+# FIXED_COLS_TYPES = [c.type for _, _, c in fixed_cols]
 
-LONG_DOC_BENCHMARK_COLS = [t.value.col_name for t in BenchmarksLongDoc]
+
+# AutoEvalColumnLongDoc = make_autoevalcolumn("AutoEvalColumnLongDoc", LongDocBenchmarks)
+# COLS_LONG_DOC = [c.name for c in fields(AutoEvalColumnLongDoc) if not c.hidden]
+# TYPES_LONG_DOC = [c.type for c in fields(AutoEvalColumnLongDoc) if not c.hidden]
+
+# Column selection

src/display/gradio_formatting.py

@@ -1,5 +1,6 @@
 import gradio as gr
 from src.envs import BENCHMARK_VERSION_LIST, LATEST_BENCHMARK_VERSION
+from src.benchmarks import QABenchmarks
 
 def get_version_dropdown():
     return gr.Dropdown(
@@ -52,7 +53,10 @@ def get_metric_dropdown(metric_list, default_metrics):
     )
 
 
-def get_domain_dropdown(domain_list, default_domains):
+def get_domain_dropdown(benchmarks, default_domains=None):
+    domain_list = list(frozenset([c.value.domain for c in list(benchmarks.value)]))
+    if default_domains is None:
+        default_domains = domain_list
     return gr.CheckboxGroup(
         choices=domain_list,
         value=default_domains,
@@ -61,10 +65,13 @@ def get_domain_dropdown(domain_list, default_domains):
     )
 
 
-def get_language_dropdown(language_list, default_languages):
+def get_language_dropdown(benchmarks, default_languages=None):
+    language_list = list(frozenset([c.value.lang for c in list(benchmarks.value)]))
+    if default_languages is None:
+        default_languages = language_list
     return gr.Dropdown(
         choices=language_list,
-        value=language_list,
+        value=default_languages,
         label="Select the languages",
         multiselect=True,
         interactive=True

src/display/gradio_listener.py

@@ -1,53 +0,0 @@
-from src.utils import update_table, update_table_long_doc
-
-
-def set_listeners(
-        task,
-        displayed_leaderboard,
-        hidden_leaderboard,
-        search_bar,
-        selected_domains,
-        selected_langs,
-        selected_rerankings,
-        show_anonymous,
-        show_revision_and_timestamp,
-
-):
-    if task == "qa":
-        update_table_func = update_table
-    elif task == "long-doc":
-        update_table_func = update_table_long_doc
-    else:
-        raise NotImplementedError
-    # Set search_bar listener
-    search_bar.submit(
-        update_table_func,
-        [
-            hidden_leaderboard,  #  hidden_leaderboard_table_for_search,
-            selected_domains,
-            selected_langs,
-            selected_rerankings,
-            search_bar,
-            show_anonymous,
-        ],
-        displayed_leaderboard
-    )
-
-    # Set column-wise listener
-    for selector in [
-        selected_domains, selected_langs, show_anonymous, show_revision_and_timestamp, selected_rerankings
-    ]:
-        selector.change(
-            update_table_func,
-            [
-                hidden_leaderboard,
-                selected_domains,
-                selected_langs,
-                selected_rerankings,
-                search_bar,
-                show_anonymous,
-                show_revision_and_timestamp
-            ],
-            displayed_leaderboard,
-            queue=True,
-        )

src/envs.py

@@ -30,4 +30,47 @@ BENCHMARK_VERSION_LIST = [
     # "AIR-Bench_24.05",
 ]
 
-LATEST_BENCHMARK_VERSION = BENCHMARK_VERSION_LIST[-1]
+LATEST_BENCHMARK_VERSION = BENCHMARK_VERSION_LIST[0]
+DEFAULT_METRIC_QA = "ndcg_at_10"
+DEFAULT_METRIC_LONG_DOC = "recall_at_10"
+METRIC_LIST = [
+    "ndcg_at_1",
+    "ndcg_at_3",
+    "ndcg_at_5",
+    "ndcg_at_10",
+    "ndcg_at_100",
+    "ndcg_at_1000",
+    "map_at_1",
+    "map_at_3",
+    "map_at_5",
+    "map_at_10",
+    "map_at_100",
+    "map_at_1000",
+    "recall_at_1",
+    "recall_at_3",
+    "recall_at_5",
+    "recall_at_10",
+    "recall_at_100",
+    "recall_at_1000",
+    "precision_at_1",
+    "precision_at_3",
+    "precision_at_5",
+    "precision_at_10",
+    "precision_at_100",
+    "precision_at_1000",
+    "mrr_at_1",
+    "mrr_at_3",
+    "mrr_at_5",
+    "mrr_at_10",
+    "mrr_at_100",
+    "mrr_at_1000"
+]
+COL_NAME_AVG = "Average ⬆️"
+COL_NAME_RETRIEVAL_MODEL = "Retrieval Method"
+COL_NAME_RERANKING_MODEL = "Reranking Model"
+COL_NAME_RETRIEVAL_MODEL_LINK = "Retrieval Model LINK"
+COL_NAME_RERANKING_MODEL_LINK = "Reranking Model LINK"
+COL_NAME_RANK = "Rank 🏆"
+COL_NAME_REVISION = "Revision"
+COL_NAME_TIMESTAMP = "Submission Date"
+COL_NAME_IS_ANONYMOUS = "Anonymous Submission"

src/loaders.py

@@ -0,0 +1,102 @@
+import os.path
+from typing import List
+
+import pandas as pd
+
+from src.envs import DEFAULT_METRIC_QA, DEFAULT_METRIC_LONG_DOC, COL_NAME_REVISION, COL_NAME_TIMESTAMP, \
+    COL_NAME_IS_ANONYMOUS, BENCHMARK_VERSION_LIST
+
+from src.models import FullEvalResult, LeaderboardDataStore
+from src.utils import get_default_cols, get_leaderboard_df
+
+pd.options.mode.copy_on_write = True
+
+
+def load_raw_eval_results(results_path: str) -> List[FullEvalResult]:
+    """
+    Load the evaluation results from a json file
+    """
+    model_result_filepaths = []
+    for root, dirs, files in os.walk(results_path):
+        if len(files) == 0:
+            continue
+
+        # select the latest results
+        for file in files:
+            if not (file.startswith("results") and file.endswith(".json")):
+                print(f'skip {file}')
+                continue
+            model_result_filepaths.append(os.path.join(root, file))
+
+    eval_results = {}
+    for model_result_filepath in model_result_filepaths:
+        # create evaluation results
+        try:
+            eval_result = FullEvalResult.init_from_json_file(model_result_filepath)
+        except UnicodeDecodeError as e:
+            print(f"loading file failed. {model_result_filepath}")
+            continue
+        print(f'file loaded: {model_result_filepath}')
+        timestamp = eval_result.timestamp
+        eval_results[timestamp] = eval_result
+
+    results = []
+    for k, v in eval_results.items():
+        try:
+            v.to_dict()
+            results.append(v)
+        except KeyError:
+            print(f"loading failed: {k}")
+            continue
+    return results
+
+def get_safe_name(name: str):
+    """Get RFC 1123 compatible safe name"""
+    name = name.replace('-', '_')
+    return ''.join(
+        character.lower()
+        for character in name
+        if (character.isalnum() or character == '_'))
+
+def load_leaderboard_datastore(file_path, version) -> LeaderboardDataStore:
+    slug = get_safe_name(version)[-4:]
+    lb_data_store = LeaderboardDataStore(version, slug, None, None, None, None, None, None, None, None)
+    lb_data_store.raw_data = load_raw_eval_results(file_path)
+    print(f'raw data: {len(lb_data_store.raw_data)}')
+
+    lb_data_store.raw_df_qa = get_leaderboard_df(
+        lb_data_store, task='qa', metric=DEFAULT_METRIC_QA)
+    print(f'QA data loaded: {lb_data_store.raw_df_qa.shape}')
+    lb_data_store.leaderboard_df_qa = lb_data_store.raw_df_qa.copy()
+    shown_columns_qa, types_qa = get_default_cols('qa', lb_data_store.slug, add_fix_cols=True)
+    # shown_columns_qa, types_qa = get_default_cols(
+    #     'qa', lb_data_store.leaderboard_df_qa.columns, add_fix_cols=True)
+    lb_data_store.types_qa = types_qa
+    lb_data_store.leaderboard_df_qa = \
+        lb_data_store.leaderboard_df_qa[~lb_data_store.leaderboard_df_qa[COL_NAME_IS_ANONYMOUS]][shown_columns_qa]
+    lb_data_store.leaderboard_df_qa.drop([COL_NAME_REVISION, COL_NAME_TIMESTAMP], axis=1, inplace=True)
+
+    lb_data_store.raw_df_long_doc = get_leaderboard_df(
+        lb_data_store, task='long-doc', metric=DEFAULT_METRIC_LONG_DOC)
+    print(f'Long-Doc data loaded: {len(lb_data_store.raw_df_long_doc)}')
+    lb_data_store.leaderboard_df_long_doc = lb_data_store.raw_df_long_doc.copy()
+    shown_columns_long_doc, types_long_doc = get_default_cols(
+        'long-doc', lb_data_store.slug, add_fix_cols=True)
+    lb_data_store.types_long_doc = types_long_doc
+    lb_data_store.leaderboard_df_long_doc = \
+        lb_data_store.leaderboard_df_long_doc[
+            ~lb_data_store.leaderboard_df_long_doc[COL_NAME_IS_ANONYMOUS]][shown_columns_long_doc]
+    lb_data_store.leaderboard_df_long_doc.drop([COL_NAME_REVISION, COL_NAME_TIMESTAMP], axis=1, inplace=True)
+
+    lb_data_store.reranking_models = sorted(
+        list(frozenset([eval_result.reranking_model for eval_result in lb_data_store.raw_data])))
+    return lb_data_store
+
+
+def load_eval_results(file_path: str):
+    output = {}
+    # versions = BENCHMARK_VERSION_LIST
+    for version in BENCHMARK_VERSION_LIST:
+        fn = f"{file_path}/{version}"
+        output[version] = load_leaderboard_datastore(fn, version)
+    return output

src/{read_evals.py → models.py}

@@ -1,38 +1,15 @@
 import json
-import os.path
 from collections import defaultdict
 from dataclasses import dataclass
-from typing import List
+from typing import List, Optional
 
 import pandas as pd
 
 from src.benchmarks import get_safe_name
-from src.display.utils import (
-    COL_NAME_RERANKING_MODEL,
-    COL_NAME_RETRIEVAL_MODEL,
-    COL_NAME_RERANKING_MODEL_LINK,
-    COL_NAME_RETRIEVAL_MODEL_LINK,
-    COL_NAME_REVISION,
-    COL_NAME_TIMESTAMP,
-    COL_NAME_IS_ANONYMOUS,
-    COLS_QA,
-    QA_BENCHMARK_COLS,
-    COLS_LONG_DOC,
-    LONG_DOC_BENCHMARK_COLS,
-    COL_NAME_AVG,
-    COL_NAME_RANK
-)
-
+from src.envs import COL_NAME_RETRIEVAL_MODEL, COL_NAME_RERANKING_MODEL, COL_NAME_RETRIEVAL_MODEL_LINK, \
+    COL_NAME_RERANKING_MODEL_LINK, COL_NAME_REVISION, COL_NAME_TIMESTAMP, COL_NAME_IS_ANONYMOUS
 from src.display.formatting import make_clickable_model
 
-pd.options.mode.copy_on_write = True
-
-def calculate_mean(row):
-    if pd.isna(row).any():
-        return -1
-    else:
-        return row.mean()
-
 
 @dataclass
 class EvalResult:
@@ -149,80 +126,15 @@ class FullEvalResult:
         return [v for v in results.values()]
 
 
-def get_raw_eval_results(results_path: str) -> List[FullEvalResult]:
-    """
-    Load the evaluation results from a json file
-    """
-    model_result_filepaths = []
-    for root, dirs, files in os.walk(results_path):
-        if len(files) == 0:
-            continue
-
-        # select the latest results
-        for file in files:
-            if not (file.startswith("results") and file.endswith(".json")):
-                print(f'skip {file}')
-                continue
-            model_result_filepaths.append(os.path.join(root, file))
-
-    eval_results = {}
-    for model_result_filepath in model_result_filepaths:
-        # create evaluation results
-        try:
-            eval_result = FullEvalResult.init_from_json_file(model_result_filepath)
-        except UnicodeDecodeError as e:
-            print(f"loading file failed. {model_result_filepath}")
-            continue
-        print(f'file loaded: {model_result_filepath}')
-        timestamp = eval_result.timestamp
-        eval_results[timestamp] = eval_result
-
-    results = []
-    for k, v in eval_results.items():
-        try:
-            v.to_dict()
-            results.append(v)
-        except KeyError:
-            print(f"loading failed: {k}")
-            continue
-    return results
-
-
-def get_leaderboard_df(raw_data: List[FullEvalResult], task: str, metric: str) -> pd.DataFrame:
-    """
-    Creates a dataframe from all the individual experiment results
-    """
-    cols = [COL_NAME_IS_ANONYMOUS, ]
-    if task == "qa":
-        cols += COLS_QA
-        benchmark_cols = QA_BENCHMARK_COLS
-    elif task == "long-doc":
-        cols += COLS_LONG_DOC
-        benchmark_cols = LONG_DOC_BENCHMARK_COLS
-    else:
-        raise NotImplemented
-    all_data_json = []
-    for v in raw_data:
-        all_data_json += v.to_dict(task=task, metric=metric)
-    df = pd.DataFrame.from_records(all_data_json)
-    # print(f'dataframe created: {df.shape}')
-
-    _benchmark_cols = frozenset(benchmark_cols).intersection(frozenset(df.columns.to_list()))
-
-    # calculate the average score for selected benchmarks
-    df[COL_NAME_AVG] = df[list(_benchmark_cols)].apply(calculate_mean, axis=1).round(decimals=2)
-    df.sort_values(by=[COL_NAME_AVG], ascending=False, inplace=True)
-    df.reset_index(inplace=True, drop=True)
-
-    _cols = frozenset(cols).intersection(frozenset(df.columns.to_list()))
-    df = df[_cols].round(decimals=2)
-
-    # filter out if any of the benchmarks have not been produced
-    df[COL_NAME_RANK] = df[COL_NAME_AVG].rank(ascending=False, method="min")
-
-    # shorten the revision
-    df[COL_NAME_REVISION] = df[COL_NAME_REVISION].str[:6]
-    
-    # # replace "0" with "-" for average score
-    # df[COL_NAME_AVG] = df[COL_NAME_AVG].replace(0, "-")
-    return df
+@dataclass
+class LeaderboardDataStore:
+    version: str
+    slug: str
+    raw_data: Optional[list]
+    raw_df_qa: Optional[pd.DataFrame]
+    raw_df_long_doc: Optional[pd.DataFrame]
+    leaderboard_df_qa: Optional[pd.DataFrame]
+    leaderboard_df_long_doc: Optional[pd.DataFrame]
+    reranking_models: Optional[list]
+    types_qa: Optional[list]
+    types_long_doc: Optional[list]

src/utils.py

@@ -2,20 +2,24 @@ import json
 import hashlib
 from datetime import datetime, timezone
 from pathlib import Path
-from typing import List
 
 import pandas as pd
 
-from src.benchmarks import BENCHMARK_COLS_QA, BENCHMARK_COLS_LONG_DOC, BenchmarksQA, BenchmarksLongDoc
+from src.benchmarks import QABenchmarks, LongDocBenchmarks
 from src.display.formatting import styled_message, styled_error
-from src.display.utils import COLS_QA, TYPES_QA, COLS_LONG_DOC, TYPES_LONG_DOC, COL_NAME_RANK, COL_NAME_AVG, \
-    COL_NAME_RERANKING_MODEL, COL_NAME_RETRIEVAL_MODEL, COL_NAME_IS_ANONYMOUS, COL_NAME_TIMESTAMP, COL_NAME_REVISION, get_default_auto_eval_column_dict
-from src.envs import API, SEARCH_RESULTS_REPO, LATEST_BENCHMARK_VERSION
-from src.read_evals import FullEvalResult, get_leaderboard_df, calculate_mean
+from src.display.columns import get_default_col_names_and_types, get_fixed_col_names_and_types
+from src.envs import API, SEARCH_RESULTS_REPO, LATEST_BENCHMARK_VERSION, COL_NAME_AVG, COL_NAME_RETRIEVAL_MODEL, \
+    COL_NAME_RERANKING_MODEL, COL_NAME_RANK, COL_NAME_REVISION, COL_NAME_TIMESTAMP, COL_NAME_IS_ANONYMOUS
 
 import re
 
 
+def calculate_mean(row):
+    if pd.isna(row).any():
+        return -1
+    else:
+        return row.mean()
+
 def remove_html(input_str):
     # Regular expression for finding HTML tags
     clean = re.sub(r'<.*?>', '', input_str)
@@ -55,67 +59,61 @@ def search_table(df: pd.DataFrame, query: str) -> pd.DataFrame:
     return df[(df[COL_NAME_RETRIEVAL_MODEL].str.contains(query, case=False))]
 
 
-def get_default_cols(task: str, columns: list=[], add_fix_cols: bool=True) -> list:
+def get_default_cols(task: str, version_slug, add_fix_cols: bool=True) -> tuple:
     cols = []
     types = []
     if task == "qa":
-        cols_list = COLS_QA
-        types_list = TYPES_QA
-        benchmark_list = BENCHMARK_COLS_QA
+        benchmarks = QABenchmarks[version_slug]
     elif task == "long-doc":
-        cols_list = COLS_LONG_DOC
-        types_list = TYPES_LONG_DOC
-        benchmark_list = BENCHMARK_COLS_LONG_DOC
+        benchmarks = LongDocBenchmarks[version_slug]
     else:
         raise NotImplemented
+    cols_list, types_list = get_default_col_names_and_types(benchmarks)
+    benchmark_list = [c.value.col_name for c in list(benchmarks.value)]
     for col_name, col_type in zip(cols_list, types_list):
         if col_name not in benchmark_list:
             continue
-        if len(columns) > 0 and col_name not in columns:
-            continue
         cols.append(col_name)
         types.append(col_type)
 
     if add_fix_cols:
         _cols = []
         _types = []
+        fixed_cols, fixed_cols_types = get_fixed_col_names_and_types()
         for col_name, col_type in zip(cols, types):
-            if col_name in FIXED_COLS:
+            if col_name in fixed_cols:
                 continue
             _cols.append(col_name)
             _types.append(col_type)
-        cols = FIXED_COLS + _cols
-        types = FIXED_COLS_TYPES + _types
+        cols = fixed_cols + _cols
+        types = fixed_cols_types + _types
     return cols, types
 
 
-fixed_cols = get_default_auto_eval_column_dict()[:-3]
-
-FIXED_COLS = [c.name for _, _, c in fixed_cols]
-FIXED_COLS_TYPES = [c.type for _, _, c in fixed_cols]
-
-
 def select_columns(
         df: pd.DataFrame,
         domain_query: list,
         language_query: list,
         task: str = "qa",
-        reset_ranking: bool = True
+        reset_ranking: bool = True,
+        version_slug: str = None
 ) -> pd.DataFrame:
-    cols, _ = get_default_cols(task=task, columns=df.columns, add_fix_cols=False)
+    cols, _ = get_default_cols(task=task, version_slug=version_slug, add_fix_cols=False)
     selected_cols = []
     for c in cols:
         if task == "qa":
-            eval_col = BenchmarksQA[c].value
+            eval_col = QABenchmarks[version_slug].value[c].value
         elif task == "long-doc":
-            eval_col = BenchmarksLongDoc[c].value
+            eval_col = LongDocBenchmarks[version_slug].value[c].value
         if eval_col.domain not in domain_query:
             continue
         if eval_col.lang not in language_query:
             continue
         selected_cols.append(c)
     # We use COLS to maintain sorting
-    filtered_df = df[FIXED_COLS + selected_cols]
+    fixed_cols, _ = get_fixed_col_names_and_types()
+    filtered_df = df[fixed_cols + selected_cols]
+    filtered_df.replace({"": pd.NA}, inplace=True)
     if reset_ranking:
         filtered_df[COL_NAME_AVG] = filtered_df[selected_cols].apply(calculate_mean, axis=1).round(decimals=2)
         filtered_df.sort_values(by=[COL_NAME_AVG], ascending=False, inplace=True)
@@ -124,9 +122,17 @@ def select_columns(
 
     return filtered_df
 
+def get_safe_name(name: str):
+    """Get RFC 1123 compatible safe name"""
+    name = name.replace('-', '_')
+    return ''.join(
+        character.lower()
+        for character in name
+        if (character.isalnum() or character == '_'))
 
 def _update_table(
         task: str,
+        version: str,
         hidden_df: pd.DataFrame,
         domains: list,
         langs: list,
@@ -136,32 +142,20 @@ def _update_table(
         reset_ranking: bool = True,
         show_revision_and_timestamp: bool = False
 ):
+    version_slug = get_safe_name(version)[-4:]
     filtered_df = hidden_df.copy()
     if not show_anonymous:
         filtered_df = filtered_df[~filtered_df[COL_NAME_IS_ANONYMOUS]]
     filtered_df = filter_models(filtered_df, reranking_query)
     filtered_df = filter_queries(query, filtered_df)
-    filtered_df = select_columns(filtered_df, domains, langs, task, reset_ranking)
+    filtered_df = select_columns(filtered_df, domains, langs, task, reset_ranking, version_slug)
     if not show_revision_and_timestamp:
         filtered_df.drop([COL_NAME_REVISION, COL_NAME_TIMESTAMP], axis=1, inplace=True)
     return filtered_df
 
 
-def update_table(
-        hidden_df: pd.DataFrame,
-        domains: list,
-        langs: list,
-        reranking_query: list,
-        query: str,
-        show_anonymous: bool,
-        show_revision_and_timestamp: bool = False,
-        reset_ranking: bool = True
-):
-    return _update_table(
-        "qa", hidden_df, domains, langs, reranking_query, query, show_anonymous, reset_ranking, show_revision_and_timestamp)
-
-
 def update_table_long_doc(
+        version: str,
         hidden_df: pd.DataFrame,
         domains: list,
         langs: list,
@@ -173,11 +167,13 @@ def update_table_long_doc(
 
 ):
     return _update_table(
-        "long-doc", hidden_df, domains, langs, reranking_query, query, show_anonymous, reset_ranking, show_revision_and_timestamp)
+        "long-doc",
+        version,
+        hidden_df, domains, langs, reranking_query, query, show_anonymous, reset_ranking, show_revision_and_timestamp)
 
 
 def update_metric(
-        raw_data: List[FullEvalResult],
+        datastore,
         task: str,
         metric: str,
         domains: list,
@@ -187,9 +183,12 @@ def update_metric(
         show_anonymous: bool = False,
         show_revision_and_timestamp: bool = False,
 ) -> pd.DataFrame:
+    # raw_data = datastore.raw_data
     if task == 'qa':
-        leaderboard_df = get_leaderboard_df(raw_data, task=task, metric=metric)
+        leaderboard_df = get_leaderboard_df(datastore, task=task, metric=metric)
+        version = datastore.version
         return update_table(
+            version,
             leaderboard_df,
             domains,
             langs,
@@ -199,8 +198,10 @@ def update_metric(
             show_revision_and_timestamp
         )
     elif task == "long-doc":
-        leaderboard_df = get_leaderboard_df(raw_data, task=task, metric=metric)
+        leaderboard_df = get_leaderboard_df(datastore, task=task, metric=metric)
+        version = datastore.version
         return update_table_long_doc(
+            version,
             leaderboard_df,
             domains,
             langs,
@@ -218,7 +219,6 @@ def upload_file(filepath: str):
     return filepath
 
 
-
 def get_iso_format_timestamp():
     # Get the current timestamp with UTC as the timezone
     current_timestamp = datetime.now(timezone.utc)
@@ -316,3 +316,95 @@ def submit_results(
 def reset_rank(df):
     df[COL_NAME_RANK] = df[COL_NAME_AVG].rank(ascending=False, method="min")
     return df
+
+
+def get_leaderboard_df(datastore, task: str, metric: str) -> pd.DataFrame:
+    """
+    Creates a dataframe from all the individual experiment results
+    """
+    raw_data = datastore.raw_data
+    cols = [COL_NAME_IS_ANONYMOUS, ]
+    if task == "qa":
+        benchmarks = QABenchmarks[datastore.slug]
+    elif task == "long-doc":
+        benchmarks = LongDocBenchmarks[datastore.slug]
+    else:
+        raise NotImplemented
+    cols_qa, _ = get_default_col_names_and_types(benchmarks)
+    cols += cols_qa
+    benchmark_cols = [t.value.col_name for t in list(benchmarks.value)]
+    all_data_json = []
+    for v in raw_data:
+        all_data_json += v.to_dict(task=task, metric=metric)
+    df = pd.DataFrame.from_records(all_data_json)
+
+    _benchmark_cols = frozenset(benchmark_cols).intersection(frozenset(df.columns.to_list()))
+
+    # calculate the average score for selected benchmarks
+    df[COL_NAME_AVG] = df[list(_benchmark_cols)].apply(calculate_mean, axis=1).round(decimals=2)
+    df.sort_values(by=[COL_NAME_AVG], ascending=False, inplace=True)
+    df.reset_index(inplace=True, drop=True)
+
+    _cols = frozenset(cols).intersection(frozenset(df.columns.to_list()))
+    df = df[_cols].round(decimals=2)
+
+    # filter out if any of the benchmarks have not been produced
+    df[COL_NAME_RANK] = df[COL_NAME_AVG].rank(ascending=False, method="min")
+
+    # shorten the revision
+    df[COL_NAME_REVISION] = df[COL_NAME_REVISION].str[:6]
+
+    # # replace "0" with "-" for average score
+    # df[COL_NAME_AVG] = df[COL_NAME_AVG].replace(0, "-")
+    return df
+
+
+def set_listeners(
+        task,
+        target_df,
+        source_df,
+        search_bar,
+        version,
+        selected_domains,
+        selected_langs,
+        selected_rerankings,
+        show_anonymous,
+        show_revision_and_timestamp,
+):
+    if task == "qa":
+        update_table_func = update_table
+    elif task == "long-doc":
+        update_table_func = update_table_long_doc
+    else:
+        raise NotImplementedError
+    selector_list = [
+        selected_domains,
+        selected_langs,
+        selected_rerankings,
+        search_bar,
+        show_anonymous
+    ]
+    search_bar_args = [source_df, version,] + selector_list
+    selector_args = [version, source_df] + selector_list + [show_revision_and_timestamp,]
+    # Set search_bar listener
+    search_bar.submit(update_table_func, search_bar_args, target_df)
+
+    # Set column-wise listener
+    for selector in selector_list:
+        selector.change(update_table_func, selector_args, target_df, queue=True,)
+
+def update_table(
+        version: str,
+        hidden_df: pd.DataFrame,
+        domains: list,
+        langs: list,
+        reranking_query: list,
+        query: str,
+        show_anonymous: bool,
+        show_revision_and_timestamp: bool = False,
+        reset_ranking: bool = True,
+):
+    return _update_table(
+        "qa",
+        version,
+        hidden_df, domains, langs, reranking_query, query, show_anonymous, reset_ranking, show_revision_and_timestamp)

tests/src/display/test_utils.py

@@ -1,5 +1,5 @@
 import pytest
-from src.display.utils import fields, AutoEvalColumnQA, COLS_QA, COLS_LONG_DOC, COLS_LITE, TYPES_QA, TYPES_LONG_DOC, QA_BENCHMARK_COLS, LONG_DOC_BENCHMARK_COLS, get_default_auto_eval_column_dict
+from src.display.utils import fields, AutoEvalColumnQA, COLS_QA, COLS_LONG_DOC, TYPES_QA, TYPES_LONG_DOC, get_default_auto_eval_column_dict
 
 
 def test_fields():
@@ -10,11 +10,8 @@ def test_fields():
 def test_macro_variables():
     print(f'COLS_QA: {COLS_QA}')
     print(f'COLS_LONG_DOC: {COLS_LONG_DOC}')
-    print(f'COLS_LITE: {COLS_LITE}')
     print(f'TYPES_QA: {TYPES_QA}')
     print(f'TYPES_LONG_DOC: {TYPES_LONG_DOC}')
-    print(f'QA_BENCHMARK_COLS: {QA_BENCHMARK_COLS}')
-    print(f'LONG_DOC_BENCHMARK_COLS: {LONG_DOC_BENCHMARK_COLS}')
 
 
 def test_get_default_auto_eval_column_dict():

tests/src/test_benchmarks.py

@@ -1,9 +1,16 @@
-from src.benchmarks import BenchmarksQA, BenchmarksLongDoc
+from src.benchmarks import QABenchmarks, LongDocBenchmarks
 
 
 def test_qabenchmarks():
-    print(list(BenchmarksQA))
+    for benchmark_list in list(QABenchmarks):
+        print(benchmark_list.name)
+        for b in list(benchmark_list.value):
+            print(b)
+    qa_benchmarks = QABenchmarks["2404"]
+    l = list(frozenset([c.value.domain for c in list(qa_benchmarks.value)]))
+    print(l)
+
 
 
 def test_longdocbenchmarks():
-    print(list(BenchmarksLongDoc))
+    print(list(LongDocBenchmarks))

tests/src/test_read_evals.py

@@ -1,6 +1,8 @@
 from pathlib import Path
 
-from src.read_evals import FullEvalResult, get_raw_eval_results, get_leaderboard_df
+from src.read_evals import load_raw_eval_results
+from src.utils import get_leaderboard_df
+from src.models import FullEvalResult
 
 cur_fp = Path(__file__)
 
@@ -29,7 +31,7 @@ def test_to_dict():
 
 def test_get_raw_eval_results():
     results_path = cur_fp.parents[2] / "toydata" / "eval_results" / "AIR-Bench_24.04"
-    results = get_raw_eval_results(results_path)
+    results = load_raw_eval_results(results_path)
     # only load the latest results
     assert len(results) == 4
     assert results[0].eval_name == "bge-base-en-v1.5_NoReranker"
@@ -40,7 +42,7 @@ def test_get_raw_eval_results():
 
 def test_get_leaderboard_df():
     results_path = cur_fp.parents[2] / "toydata" / "eval_results" / "AIR-Bench_24.04"
-    raw_data = get_raw_eval_results(results_path)
+    raw_data = load_raw_eval_results(results_path)
     df = get_leaderboard_df(raw_data, 'qa', 'ndcg_at_10')
     assert df.shape[0] == 4
     # the results contain only one embedding model
@@ -55,7 +57,7 @@ def test_get_leaderboard_df():
 
 def test_get_leaderboard_df_long_doc():
     results_path = cur_fp.parents[2] / "toydata" / "test_results"
-    raw_data = get_raw_eval_results(results_path)
+    raw_data = load_raw_eval_results(results_path)
     df = get_leaderboard_df(raw_data, 'long-doc', 'ndcg_at_1')
     assert df.shape[0] == 2
     # the results contain only one embedding model

tests/test_utils.py

@@ -1,8 +1,10 @@
 import pandas as pd
 import pytest
 
-from src.utils import filter_models, search_table, filter_queries, select_columns, update_table_long_doc, get_iso_format_timestamp, get_default_cols, update_table
-from src.display.utils import COL_NAME_IS_ANONYMOUS, COL_NAME_REVISION, COL_NAME_TIMESTAMP, COL_NAME_RERANKING_MODEL, COL_NAME_RETRIEVAL_MODEL, COL_NAME_RANK, COL_NAME_AVG
+from src.utils import filter_models, search_table, filter_queries, select_columns, update_table_long_doc, get_iso_format_timestamp, get_default_cols
+from app import update_table
+from src.envs import COL_NAME_AVG, COL_NAME_RETRIEVAL_MODEL, COL_NAME_RERANKING_MODEL, COL_NAME_RANK, COL_NAME_REVISION, \
+    COL_NAME_TIMESTAMP, COL_NAME_IS_ANONYMOUS
 
 
 @pytest.fixture