feat: add versioning for the qa retrieval

app.py

@@ -14,7 +14,7 @@ from src.display.css_html_js import custom_css
 from src.envs import (
     API,
     EVAL_RESULTS_PATH,
-    REPO_ID, DEFAULT_METRIC_QA, DEFAULT_METRIC_LONG_DOC, METRIC_LIST, LATEST_BENCHMARK_VERSION, COL_NAME_RERANKING_MODEL
+    REPO_ID, DEFAULT_METRIC_QA, DEFAULT_METRIC_LONG_DOC, METRIC_LIST, LATEST_BENCHMARK_VERSION, COL_NAME_RERANKING_MODEL, COL_NAME_RETRIEVAL_MODEL, BM25_LINK
 )
 from src.loaders import (
     load_eval_results
@@ -22,7 +22,8 @@ from src.loaders import (
 from src.utils import (
     update_metric,
     set_listeners,
-    reset_rank
+    reset_rank,
+    remove_html
 )
 from src.display.gradio_formatting import (
     get_version_dropdown,
@@ -183,6 +184,7 @@ with demo:
                         lb_df_retriever = datastore.leaderboard_df_qa[datastore.leaderboard_df_qa[COL_NAME_RERANKING_MODEL] == "NoReranker"]
                         lb_df_retriever = reset_rank(lb_df_retriever)
                         lb_table_retriever = get_leaderboard_table(lb_df_retriever, datastore.types_qa)
+
                         # Dummy leaderboard for handling the case when the user uses backspace key
                         hidden_lb_df_retriever = datastore.raw_df_qa[datastore.raw_df_qa[COL_NAME_RERANKING_MODEL] == "NoReranker"]
                         hidden_lb_df_retriever = reset_rank(hidden_lb_df_retriever)
@@ -222,9 +224,8 @@ with demo:
                             lb_table_retriever,
                             queue=True
                         )
-                    """
                     with gr.TabItem("Reranking Only", id=12):
-                        lb_df_reranker = data["AIR-Bench_24.04"].leaderboard_df_qa[data["AIR-Bench_24.04"].leaderboard_df_qa[COL_NAME_RETRIEVAL_MODEL] == BM25_LINK]
+                        lb_df_reranker = datastore.leaderboard_df_qa[datastore.leaderboard_df_qa[COL_NAME_RETRIEVAL_MODEL] == BM25_LINK]
                         lb_df_reranker = reset_rank(lb_df_reranker)
                         reranking_models_reranker = lb_df_reranker[COL_NAME_RERANKING_MODEL].apply(remove_html).unique().tolist()
                         with gr.Row():
@@ -232,11 +233,18 @@ with demo:
                                 selected_rerankings_reranker = get_reranking_dropdown(reranking_models_reranker)
                             with gr.Column(scale=1):
                                 search_bar_reranker = gr.Textbox(show_label=False, visible=False)
-                        lb_table_reranker = get_leaderboard_table(lb_df_reranker, data["AIR-Bench_24.04"].types_qa)
-                        hidden_lb_df_reranker = data["AIR-Bench_24.04"].raw_df_qa[data["AIR-Bench_24.04"].raw_df_qa[COL_NAME_RETRIEVAL_MODEL] == BM25_LINK]
+                        lb_table_reranker = get_leaderboard_table(lb_df_reranker, datastore.types_qa)
+
+                        hidden_lb_df_reranker = datastore.raw_df_qa[datastore.raw_df_qa[COL_NAME_RETRIEVAL_MODEL] == BM25_LINK]
                         hidden_lb_df_reranker = reset_rank(hidden_lb_df_reranker)
                         hidden_lb_table_reranker = get_leaderboard_table(
-                            hidden_lb_df_reranker, data["AIR-Bench_24.04"].types_qa, visible=False
+                            hidden_lb_df_reranker, datastore.types_qa, visible=False
+                        )
+
+                        selected_version.change(
+                            update_datastore,
+                            [selected_version,],
+                            [selected_domains, selected_langs, selected_rerankings_reranker, lb_table_reranker, hidden_lb_table_reranker]
                         )
 
                         set_listeners(
@@ -244,6 +252,7 @@ with demo:
                             lb_table_reranker,
                             hidden_lb_table_reranker,
                             search_bar_reranker,
+                            selected_version,
                             selected_domains,
                             selected_langs,
                             selected_rerankings_reranker,
@@ -261,11 +270,11 @@ with demo:
                                 search_bar_reranker,
                                 show_anonymous,
                                 show_revision_and_timestamp,
-                                selected_version,
                             ],
                             lb_table_reranker,
                             queue=True
                         )
+"""
             with gr.TabItem("Long Doc", elem_id="long-doc-benchmark-tab-table", id=1):
                 with gr.Row():
                     with gr.Column(min_width=320):

src/envs.py

@@ -30,7 +30,7 @@ BENCHMARK_VERSION_LIST = [
     "AIR-Bench_24.05",
 ]
 
-LATEST_BENCHMARK_VERSION = BENCHMARK_VERSION_LIST[-1]
+LATEST_BENCHMARK_VERSION = BENCHMARK_VERSION_LIST[0]
 DEFAULT_METRIC_QA = "ndcg_at_10"
 DEFAULT_METRIC_LONG_DOC = "recall_at_10"
 METRIC_LIST = [

src/utils.py

@@ -177,10 +177,6 @@ def _update_table(
         show_revision_and_timestamp: bool = False
 ):
     version_slug = get_safe_name(version)[-4:]
-    if isinstance(hidden_df, str):
-        print(f"task: {task}")
-        print(f"version: {version}")
-        print(f"hidden_df is a string: {hidden_df}")
     filtered_df = hidden_df.copy()
     if not show_anonymous:
         filtered_df = filtered_df[~filtered_df[COL_NAME_IS_ANONYMOUS]]
@@ -257,7 +253,6 @@ def upload_file(filepath: str):
     return filepath
 
 
-
 def get_iso_format_timestamp():
     # Get the current timestamp with UTC as the timezone
     current_timestamp = datetime.now(timezone.utc)
@@ -377,7 +372,6 @@ def get_leaderboard_df(datastore, task: str, metric: str) -> pd.DataFrame:
     for v in raw_data:
         all_data_json += v.to_dict(task=task, metric=metric)
     df = pd.DataFrame.from_records(all_data_json)
-    # print(f'dataframe created: {df.shape}')
 
     _benchmark_cols = frozenset(benchmark_cols).intersection(frozenset(df.columns.to_list()))