style: reformat the styles

Makefile

@@ -3,14 +3,20 @@
 
 style:
 	python -m black --line-length 119 .
+	python -m black --line-length 119 src
 	python -m isort .
+	python -m isort src
 	ruff check --fix .
+	ruff check --fix src
 
 
 quality:
 	python -m black --check --line-length 119 .
+	python -m black --check --line-length 119 src
 	python -m isort --check-only .
+	python -m isort --check-only src
 	ruff check .
+	ruff check src
 
 
 test:

app.py

@@ -63,13 +63,13 @@ datastore = ds_dict[LATEST_BENCHMARK_VERSION]
 
 
 def update_qa_metric(
-        metric: str,
-        domains: list,
-        langs: list,
-        reranking_model: list,
-        query: str,
-        show_anonymous: bool,
-        show_revision_and_timestamp: bool,
+    metric: str,
+    domains: list,
+    langs: list,
+    reranking_model: list,
+    query: str,
+    show_anonymous: bool,
+    show_revision_and_timestamp: bool,
 ):
     global datastore
     return update_metric(
@@ -86,13 +86,13 @@ def update_qa_metric(
 
 
 def update_doc_metric(
-        metric: str,
-        domains: list,
-        langs: list,
-        reranking_model: list,
-        query: str,
-        show_anonymous: bool,
-        show_revision_and_timestamp,
+    metric: str,
+    domains: list,
+    langs: list,
+    reranking_model: list,
+    query: str,
+    show_anonymous: bool,
+    show_revision_and_timestamp,
 ):
     global datastore
     return update_metric(
@@ -218,7 +218,7 @@ with demo:
                         # Dummy leaderboard for handling the case when the user uses backspace key
                         _qa_df_ret_hidden = datastore.qa_raw_df[
                             datastore.qa_raw_df[COL_NAME_RERANKING_MODEL] == "NoReranker"
-                            ]
+                        ]
                         _qa_df_ret_hidden = reset_rank(_qa_df_ret_hidden)
                         qa_df_elem_ret_hidden = get_leaderboard_table(
                             _qa_df_ret_hidden, datastore.qa_types, visible=False
@@ -277,7 +277,7 @@ with demo:
 
                         _qa_df_rerank_hidden = datastore.qa_raw_df[
                             datastore.qa_raw_df[COL_NAME_RETRIEVAL_MODEL] == BM25_LINK
-                            ]
+                        ]
                         _qa_df_rerank_hidden = reset_rank(_qa_df_rerank_hidden)
                         qa_df_elem_rerank_hidden = get_leaderboard_table(
                             _qa_df_rerank_hidden, datastore.qa_types, visible=False
@@ -391,13 +391,13 @@ with demo:
 
                         _doc_df_ret = datastore.doc_fmt_df[
                             datastore.doc_fmt_df[COL_NAME_RERANKING_MODEL] == "NoReranker"
-                            ]
+                        ]
                         _doc_df_ret = reset_rank(_doc_df_ret)
                         doc_df_elem_ret = get_leaderboard_table(_doc_df_ret, datastore.doc_types)
 
                         _doc_df_ret_hidden = datastore.doc_raw_df[
                             datastore.doc_raw_df[COL_NAME_RERANKING_MODEL] == "NoReranker"
-                            ]
+                        ]
                         _doc_df_ret_hidden = reset_rank(_doc_df_ret_hidden)
                         doc_df_elem_ret_hidden = get_leaderboard_table(
                             _doc_df_ret_hidden, datastore.doc_types, visible=False
@@ -439,7 +439,7 @@ with demo:
                     with gr.TabItem("Reranking Only", id=22):
                         _doc_df_rerank = datastore.doc_fmt_df[
                             datastore.doc_fmt_df[COL_NAME_RETRIEVAL_MODEL] == BM25_LINK
-                            ]
+                        ]
                         _doc_df_rerank = reset_rank(_doc_df_rerank)
                         doc_rerank_models = (
                             _doc_df_rerank[COL_NAME_RERANKING_MODEL].apply(remove_html).unique().tolist()
@@ -452,7 +452,7 @@ with demo:
                         doc_df_elem_rerank = get_leaderboard_table(_doc_df_rerank, datastore.doc_types)
                         _doc_df_rerank_hidden = datastore.doc_raw_df[
                             datastore.doc_raw_df[COL_NAME_RETRIEVAL_MODEL] == BM25_LINK
-                            ]
+                        ]
                         _doc_df_rerank_hidden = reset_rank(_doc_df_rerank_hidden)
                         doc_df_elem_rerank_hidden = get_leaderboard_table(
                             _doc_df_rerank_hidden, datastore.doc_types, visible=False

pyproject.toml

@@ -1,9 +1,9 @@
 [tool.ruff]
 # Enable pycodestyle (`E`) and Pyflakes (`F`) codes by default.
-select = ["E", "F"]
-ignore = ["E501"] # line too long (black is taking care of this)
+lint.select = ["E", "F"]
+lint.ignore = ["E501"] # line too long (black is taking care of this)
 line-length = 119
-fixable = ["A", "B", "C", "D", "E", "F", "G", "I", "N", "Q", "S", "T", "W", "ANN", "ARG", "BLE", "COM", "DJ", "DTZ", "EM", "ERA", "EXE", "FBT", "ICN", "INP", "ISC", "NPY", "PD", "PGH", "PIE", "PL", "PT", "PTH", "PYI", "RET", "RSE", "RUF", "SIM", "SLF", "TCH", "TID", "TRY", "UP", "YTT"]
+lint.fixable = ["A", "B", "C", "D", "E", "F", "G", "I", "N", "Q", "S", "T", "W", "ANN", "ARG", "BLE", "COM", "DJ", "DTZ", "EM", "ERA", "EXE", "FBT", "ICN", "INP", "ISC", "NPY", "PD", "PGH", "PIE", "PL", "PT", "PTH", "PYI", "RET", "RSE", "RUF", "SIM", "SLF", "TCH", "TID", "TRY", "UP", "YTT"]
 
 [tool.isort]
 profile = "black"

src/benchmarks.py

@@ -30,9 +30,7 @@ def get_qa_benchmarks_dict(version: str):
                 for metric in dataset_list:
                     if "test" not in dataset_list[metric]["splits"]:
                         continue
-                    benchmark_dict[benchmark_name] = Benchmark(
-                        benchmark_name, metric, col_name, domain, lang, task
-                    )
+                    benchmark_dict[benchmark_name] = Benchmark(benchmark_name, metric, col_name, domain, lang, task)
     return benchmark_dict
 
 
@@ -59,20 +57,14 @@ def get_doc_benchmarks_dict(version: str):
 _qa_benchmark_dict = {}
 for version in BENCHMARK_VERSION_LIST:
     safe_version_name = get_safe_name(version)
-    _qa_benchmark_dict[safe_version_name] = \
-        Enum(
-            f"QABenchmarks_{safe_version_name}",
-            get_qa_benchmarks_dict(version)
-        )
+    _qa_benchmark_dict[safe_version_name] = Enum(f"QABenchmarks_{safe_version_name}", get_qa_benchmarks_dict(version))
 
 _doc_benchmark_dict = {}
 for version in BENCHMARK_VERSION_LIST:
     safe_version_name = get_safe_name(version)
-    _doc_benchmark_dict[safe_version_name] = \
-        Enum(
-            f"LongDocBenchmarks_{safe_version_name}",
-            get_doc_benchmarks_dict(version)
-        )
+    _doc_benchmark_dict[safe_version_name] = Enum(
+        f"LongDocBenchmarks_{safe_version_name}", get_doc_benchmarks_dict(version)
+    )
 
 
 QABenchmarks = Enum("QABenchmarks", _qa_benchmark_dict)

src/columns.py

@@ -19,69 +19,28 @@ class ColumnContent:
 
 def get_default_auto_eval_column_dict():
     auto_eval_column_dict = []
-    auto_eval_column_dict.append(
-        [
-            "rank",
-            ColumnContent,
-            ColumnContent(
-                COL_NAME_RANK,
-                "number",
-                True
-            )
-        ]
-    )
+    auto_eval_column_dict.append(["rank", ColumnContent, ColumnContent(COL_NAME_RANK, "number", True)])
     auto_eval_column_dict.append(
         [
             "retrieval_model",
             ColumnContent,
-            ColumnContent(
-                COL_NAME_RETRIEVAL_MODEL,
-                "markdown",
-                True,
-                never_hidden=True
-            )
+            ColumnContent(COL_NAME_RETRIEVAL_MODEL, "markdown", True, never_hidden=True),
         ]
     )
     auto_eval_column_dict.append(
         [
             "reranking_model",
             ColumnContent,
-            ColumnContent(
-                COL_NAME_RERANKING_MODEL,
-                "markdown",
-                True,
-                never_hidden=True
-            )
+            ColumnContent(COL_NAME_RERANKING_MODEL, "markdown", True, never_hidden=True),
         ]
     )
     auto_eval_column_dict.append(
-        [
-            "revision",
-            ColumnContent,
-            ColumnContent(
-                COL_NAME_REVISION,
-                "markdown",
-                True,
-                never_hidden=True
-            )
-        ]
+        ["revision", ColumnContent, ColumnContent(COL_NAME_REVISION, "markdown", True, never_hidden=True)]
     )
     auto_eval_column_dict.append(
-        [
-            "timestamp",
-            ColumnContent,
-            ColumnContent(
-                COL_NAME_TIMESTAMP, "date", True, never_hidden=True
-            )
-        ]
-    )
-    auto_eval_column_dict.append(
-        [
-            "average",
-            ColumnContent,
-            ColumnContent(COL_NAME_AVG, "number", True)
-        ]
+        ["timestamp", ColumnContent, ColumnContent(COL_NAME_TIMESTAMP, "date", True, never_hidden=True)]
     )
+    auto_eval_column_dict.append(["average", ColumnContent, ColumnContent(COL_NAME_AVG, "number", True)])
     auto_eval_column_dict.append(
         [
             "retrieval_model_link",
@@ -91,7 +50,7 @@ def get_default_auto_eval_column_dict():
                 "markdown",
                 False,
                 hidden=True,
-            )
+            ),
         ]
     )
     auto_eval_column_dict.append(
@@ -103,20 +62,11 @@ def get_default_auto_eval_column_dict():
                 "markdown",
                 False,
                 hidden=True,
-            )
+            ),
         ]
     )
     auto_eval_column_dict.append(
-        [
-            "is_anonymous",
-            ColumnContent,
-            ColumnContent(
-                COL_NAME_IS_ANONYMOUS,
-                "bool",
-                False,
-                hidden=True
-            )
-        ]
+        ["is_anonymous", ColumnContent, ColumnContent(COL_NAME_IS_ANONYMOUS, "bool", False, hidden=True)]
     )
     return auto_eval_column_dict
 

src/envs.py

@@ -6,7 +6,9 @@ from huggingface_hub import HfApi
 # ----------------------------------
 TOKEN = os.environ.get("TOKEN", "")  # A read/write token for your org
 
-OWNER = "AIR-Bench"  # Change to your org - don't forget to create a results and request dataset, with the correct format!
+OWNER = (
+    "AIR-Bench"  # Change to your org - don't forget to create a results and request dataset, with the correct format!
+)
 # ----------------------------------
 
 REPO_ID = f"{OWNER}/leaderboard"

src/loaders.py

@@ -1,7 +1,6 @@
 import os.path
 from pathlib import Path
-from typing import Union
-from typing import Dict, List
+from typing import Dict, List, Union
 
 import pandas as pd
 

src/models.py

@@ -2,7 +2,7 @@ import json
 from collections import defaultdict
 from dataclasses import dataclass
 from enum import Enum
-from typing import List, Optional
+from typing import List
 
 import pandas as pd
 

src/utils.py

@@ -118,39 +118,36 @@ def get_selected_cols(task, version_slug, domains, languages):
 
 
 def select_columns(
-        df: pd.DataFrame,
-        domains: list,
-        languages: list,
-        task: TaskType = TaskType.qa,
-        reset_ranking: bool = True,
-        version_slug: str = None,
+    df: pd.DataFrame,
+    domains: list,
+    languages: list,
+    task: TaskType = TaskType.qa,
+    reset_ranking: bool = True,
+    version_slug: str = None,
 ) -> pd.DataFrame:
-    selected_cols = get_selected_cols(
-        task, version_slug, domains, languages)
+    selected_cols = get_selected_cols(task, version_slug, domains, languages)
     fixed_cols, _ = get_fixed_col_names_and_types()
     filtered_df = df[fixed_cols + selected_cols]
     filtered_df.replace({"": pd.NA}, inplace=True)
     if reset_ranking:
-        filtered_df[COL_NAME_AVG] = \
-            filtered_df[selected_cols].apply(calculate_mean, axis=1).round(decimals=2)
-        filtered_df.sort_values(
-            by=[COL_NAME_AVG], ascending=False, inplace=True)
+        filtered_df[COL_NAME_AVG] = filtered_df[selected_cols].apply(calculate_mean, axis=1).round(decimals=2)
+        filtered_df.sort_values(by=[COL_NAME_AVG], ascending=False, inplace=True)
         filtered_df.reset_index(inplace=True, drop=True)
         filtered_df = reset_rank(filtered_df)
     return filtered_df
 
 
 def _update_df_elem(
-        task: TaskType,
-        version: str,
-        source_df: pd.DataFrame,
-        domains: list,
-        langs: list,
-        reranking_query: list,
-        query: str,
-        show_anonymous: bool,
-        reset_ranking: bool = True,
-        show_revision_and_timestamp: bool = False,
+    task: TaskType,
+    version: str,
+    source_df: pd.DataFrame,
+    domains: list,
+    langs: list,
+    reranking_query: list,
+    query: str,
+    show_anonymous: bool,
+    reset_ranking: bool = True,
+    show_revision_and_timestamp: bool = False,
 ):
     filtered_df = source_df.copy()
     if not show_anonymous:
@@ -164,15 +161,15 @@ def _update_df_elem(
 
 
 def update_doc_df_elem(
-        version: str,
-        hidden_df: pd.DataFrame,
-        domains: list,
-        langs: list,
-        reranking_query: list,
-        query: str,
-        show_anonymous: bool,
-        show_revision_and_timestamp: bool = False,
-        reset_ranking: bool = True,
+    version: str,
+    hidden_df: pd.DataFrame,
+    domains: list,
+    langs: list,
+    reranking_query: list,
+    query: str,
+    show_anonymous: bool,
+    show_revision_and_timestamp: bool = False,
+    reset_ranking: bool = True,
 ):
     return _update_df_elem(
         TaskType.long_doc,
@@ -189,15 +186,15 @@ def update_doc_df_elem(
 
 
 def update_metric(
-        datastore,
-        task: TaskType,
-        metric: str,
-        domains: list,
-        langs: list,
-        reranking_model: list,
-        query: str,
-        show_anonymous: bool = False,
-        show_revision_and_timestamp: bool = False,
+    datastore,
+    task: TaskType,
+    metric: str,
+    domains: list,
+    langs: list,
+    reranking_model: list,
+    query: str,
+    show_anonymous: bool = False,
+    show_revision_and_timestamp: bool = False,
 ) -> pd.DataFrame:
     if task == TaskType.qa:
         update_func = update_qa_df_elem
@@ -253,13 +250,13 @@ def calculate_file_md5(file_path):
 
 
 def submit_results(
-        filepath: str,
-        model: str,
-        model_url: str,
-        reranking_model: str = "",
-        reranking_model_url: str = "",
-        version: str = LATEST_BENCHMARK_VERSION,
-        is_anonymous=False,
+    filepath: str,
+    model: str,
+    model_url: str,
+    reranking_model: str = "",
+    reranking_model_url: str = "",
+    version: str = LATEST_BENCHMARK_VERSION,
+    is_anonymous=False,
 ):
     if not filepath.endswith(".zip"):
         return styled_error(f"file uploading aborted. wrong file type: {filepath}")
@@ -355,11 +352,7 @@ def get_leaderboard_df(datastore, task: TaskType, metric: str) -> pd.DataFrame:
         benchmark_cols.append(t.value.col_name)
 
     # filter out the columns that are not in the data
-    df[COL_NAME_AVG] = (
-        df[list(benchmark_cols)]
-        .apply(calculate_mean, axis=1)
-        .round(decimals=2)
-    )
+    df[COL_NAME_AVG] = df[list(benchmark_cols)].apply(calculate_mean, axis=1).round(decimals=2)
     df.sort_values(by=[COL_NAME_AVG], ascending=False, inplace=True)
     df.reset_index(inplace=True, drop=True)
 
@@ -381,16 +374,16 @@ def get_leaderboard_df(datastore, task: TaskType, metric: str) -> pd.DataFrame:
 
 
 def set_listeners(
-        task: TaskType,
-        target_df,
-        source_df,
-        search_bar,
-        version,
-        selected_domains,
-        selected_langs,
-        selected_rerankings,
-        show_anonymous,
-        show_revision_and_timestamp,
+    task: TaskType,
+    target_df,
+    source_df,
+    search_bar,
+    version,
+    selected_domains,
+    selected_langs,
+    selected_rerankings,
+    show_anonymous,
+    show_revision_and_timestamp,
 ):
     if task == TaskType.qa:
         update_table_func = update_qa_df_elem
@@ -400,15 +393,15 @@ def set_listeners(
         raise NotImplementedError
     selector_list = [selected_domains, selected_langs, selected_rerankings, search_bar, show_anonymous]
     search_bar_args = [
-                          source_df,
-                          version,
-                      ] + selector_list
+        source_df,
+        version,
+    ] + selector_list
     selector_args = (
-            [version, source_df]
-            + selector_list
-            + [
-                show_revision_and_timestamp,
-            ]
+        [version, source_df]
+        + selector_list
+        + [
+            show_revision_and_timestamp,
+        ]
     )
     # Set search_bar listener
     search_bar.submit(update_table_func, search_bar_args, target_df)
@@ -424,15 +417,15 @@ def set_listeners(
 
 
 def update_qa_df_elem(
-        version: str,
-        hidden_df: pd.DataFrame,
-        domains: list,
-        langs: list,
-        reranking_query: list,
-        query: str,
-        show_anonymous: bool,
-        show_revision_and_timestamp: bool = False,
-        reset_ranking: bool = True,
+    version: str,
+    hidden_df: pd.DataFrame,
+    domains: list,
+    langs: list,
+    reranking_query: list,
+    query: str,
+    show_anonymous: bool,
+    show_revision_and_timestamp: bool = False,
+    reset_ranking: bool = True,
 ):
     return _update_df_elem(
         TaskType.qa,

tests/src/test_benchmarks.py

@@ -3,7 +3,6 @@ import pytest
 from src.benchmarks import LongDocBenchmarks, QABenchmarks
 from src.envs import BENCHMARK_VERSION_LIST
 
-
 # Ref: https://github.com/AIR-Bench/AIR-Bench/blob/4b27b8a8f2047a963805fcf6fb9d74be51ec440c/docs/available_tasks.md
 # 24.05
 # | Task | dev | test |
@@ -17,15 +16,8 @@ from src.envs import BENCHMARK_VERSION_LIST
 # | Long-Doc | 15 |
 # | QA | 13 |
 
-@pytest.mark.parametrize(
-    "num_datasets_dict",
-    [
-        {
-            "air_bench_2404": 13,
-            "air_bench_2405": 53
-        }
-    ]
-)
+
+@pytest.mark.parametrize("num_datasets_dict", [{"air_bench_2404": 13, "air_bench_2405": 53}])
 def test_qa_benchmarks(num_datasets_dict):
     assert len(QABenchmarks) == len(BENCHMARK_VERSION_LIST)
     for benchmark_list in list(QABenchmarks):
@@ -33,15 +25,7 @@ def test_qa_benchmarks(num_datasets_dict):
         assert num_datasets_dict[version_slug] == len(benchmark_list.value)
 
 
-@pytest.mark.parametrize(
-    "num_datasets_dict",
-    [
-        {
-            "air_bench_2404": 15,
-            "air_bench_2405": 11
-        }
-    ]
-)
+@pytest.mark.parametrize("num_datasets_dict", [{"air_bench_2404": 15, "air_bench_2405": 11}])
 def test_doc_benchmarks(num_datasets_dict):
     assert len(LongDocBenchmarks) == len(BENCHMARK_VERSION_LIST)
     for benchmark_list in list(LongDocBenchmarks):

tests/src/test_columns.py

@@ -1,12 +1,18 @@
 import pytest
 
-from src.benchmarks import QABenchmarks, LongDocBenchmarks
-from src.columns import get_default_auto_eval_column_dict, \
-    get_fixed_col_names_and_types, get_default_col_names_and_types, make_autoevalcolumn, COL_NAME_RANK, \
-    COL_NAME_RETRIEVAL_MODEL, \
-    COL_NAME_RERANKING_MODEL, COL_NAME_REVISION, \
-    COL_NAME_TIMESTAMP, COL_NAME_AVG
-
+from src.benchmarks import LongDocBenchmarks, QABenchmarks
+from src.columns import (
+    COL_NAME_AVG,
+    COL_NAME_RANK,
+    COL_NAME_RERANKING_MODEL,
+    COL_NAME_RETRIEVAL_MODEL,
+    COL_NAME_REVISION,
+    COL_NAME_TIMESTAMP,
+    get_default_auto_eval_column_dict,
+    get_default_col_names_and_types,
+    get_fixed_col_names_and_types,
+    make_autoevalcolumn,
+)
 
 # Ref: https://github.com/AIR-Bench/AIR-Bench/blob/4b27b8a8f2047a963805fcf6fb9d74be51ec440c/docs/available_tasks.md
 # 24.05
@@ -21,6 +27,7 @@ from src.columns import get_default_auto_eval_column_dict, \
 # | Long-Doc | 15 |
 # | QA | 13 |
 
+
 @pytest.fixture()
 def expected_col_names():
     return [
@@ -45,8 +52,7 @@ def expected_hidden_col_names():
     ]
 
 
-def test_get_default_auto_eval_column_dict(
-        expected_col_names, expected_hidden_col_names):
+def test_get_default_auto_eval_column_dict(expected_col_names, expected_hidden_col_names):
     col_list = get_default_auto_eval_column_dict()
     assert len(col_list) == 9
     hidden_cols = []
@@ -76,14 +82,13 @@ def test_get_fixed_col_names_and_types():
 
 
 @pytest.mark.parametrize(
-    'benchmarks, expected_benchmark_len',
+    "benchmarks, expected_benchmark_len",
     [
         (QABenchmarks, {"air_bench_2404": 13, "air_bench_2405": 53}),
-        (LongDocBenchmarks, {"air_bench_2404": 15, "air_bench_2405": 11})
-    ]
+        (LongDocBenchmarks, {"air_bench_2404": 15, "air_bench_2405": 11}),
+    ],
 )
-def test_make_autoevalcolumn(
-        benchmarks, expected_benchmark_len, expected_col_names):
+def test_make_autoevalcolumn(benchmarks, expected_benchmark_len, expected_col_names):
     expected_default_attrs = frozenset(expected_col_names)
     for benchmark in benchmarks:
         TestEvalColumn = make_autoevalcolumn("TestEvalColumn", benchmark)
@@ -98,17 +103,15 @@ def test_make_autoevalcolumn(
 
 
 @pytest.mark.parametrize(
-    'benchmarks, expected_benchmark_len',
+    "benchmarks, expected_benchmark_len",
     [
         (QABenchmarks, {"air_bench_2404": 13, "air_bench_2405": 53}),
-        (LongDocBenchmarks, {"air_bench_2404": 15, "air_bench_2405": 11})
-    ]
+        (LongDocBenchmarks, {"air_bench_2404": 15, "air_bench_2405": 11}),
+    ],
 )
 def test_get_default_col_names_and_types(
-        benchmarks,
-        expected_benchmark_len,
-        expected_col_names,
-        expected_hidden_col_names):
+    benchmarks, expected_benchmark_len, expected_col_names, expected_hidden_col_names
+):
     default_col_len = len(expected_col_names)
     hidden_col_len = len(expected_hidden_col_names)
     for benchmark in benchmarks:

tests/src/test_envs.py

@@ -1,13 +1,12 @@
 from air_benchmark.tasks import BenchmarkTable
 
-from src.envs import BENCHMARK_VERSION_LIST, DEFAULT_METRIC_QA, DEFAULT_METRIC_LONG_DOC, METRIC_LIST
+from src.envs import BENCHMARK_VERSION_LIST, DEFAULT_METRIC_LONG_DOC, DEFAULT_METRIC_QA, METRIC_LIST
 
 
 def test_benchmark_version_list():
     leaderboard_versions = frozenset(BENCHMARK_VERSION_LIST)
     available_versions = frozenset([k for k in BenchmarkTable.keys()])
-    assert leaderboard_versions.issubset(
-        available_versions)
+    assert leaderboard_versions.issubset(available_versions)
 
 
 def test_default_metrics():

tests/src/test_loaders.py

@@ -1,41 +1,34 @@
+from pathlib import Path
+
 import pandas as pd
 import pytest
-from pathlib import Path
 
-from src.loaders import load_raw_eval_results, load_leaderboard_datastore, load_eval_results
+from src.loaders import load_eval_results, load_leaderboard_datastore, load_raw_eval_results
 
 cur_fp = Path(__file__)
 
 
-@pytest.mark.parametrize(
-    "version",
-    ["AIR-Bench_24.04", "AIR-Bench_24.05"]
-)
+@pytest.mark.parametrize("version", ["AIR-Bench_24.04", "AIR-Bench_24.05"])
 def test_load_raw_eval_results(version):
-    raw_data = load_raw_eval_results(
-        cur_fp.parents[1] / f"toydata/eval_results/{version}"
-    )
+    raw_data = load_raw_eval_results(cur_fp.parents[1] / f"toydata/eval_results/{version}")
     assert len(raw_data) == 1
     full_eval_result = raw_data[0]
     expected_attr = [
-        'eval_name',
-        'retrieval_model',
-        'reranking_model',
-        'retrieval_model_link',
-        'reranking_model_link',
-        'results',
-        'timestamp',
-        'revision',
-        'is_anonymous'
+        "eval_name",
+        "retrieval_model",
+        "reranking_model",
+        "retrieval_model_link",
+        "reranking_model_link",
+        "results",
+        "timestamp",
+        "revision",
+        "is_anonymous",
     ]
     result_attr = [k for k in full_eval_result.__dict__.keys() if k[:2] != "__" and k[-2:] != "__"]
     assert sorted(expected_attr) == sorted(result_attr)
 
 
-@pytest.mark.parametrize(
-    "version",
-    ["AIR-Bench_24.04", "AIR-Bench_24.05"]
-)
+@pytest.mark.parametrize("version", ["AIR-Bench_24.04", "AIR-Bench_24.05"])
 def test_load_leaderboard_datastore(version):
     file_path = cur_fp.parents[1] / f"toydata/eval_results/{version}"
     datastore = load_leaderboard_datastore(file_path, version)
@@ -51,4 +44,3 @@ def test_load_eval_results():
     file_path = cur_fp.parents[1] / "toydata/eval_results/"
     datastore_dict = load_eval_results(file_path)
     assert len(datastore_dict) == 2
-

tests/src/test_models.py

@@ -1,6 +1,7 @@
-import pytest
 from pathlib import Path
 
+import pytest
+
 from src.models import EvalResult, FullEvalResult
 
 cur_fp = Path(__file__)
@@ -23,19 +24,13 @@ NUM_DOC_BENCHMARKS_24_05 = 11
 NUM_QA_BENCHMARKS_24_04 = 13
 NUM_DOC_BENCHMARKS_24_04 = 15
 
+
 def test_eval_result():
-    eval_result = EvalResult(
+    EvalResult(
         eval_name="eval_name",
         retrieval_model="bge-m3",
         reranking_model="NoReranking",
-        results=[
-            {
-                "domain": "law",
-                "lang": "en",
-                "dataset": "lex_files_500K-600K",
-                "value": 0.45723
-            }
-        ],
+        results=[{"domain": "law", "lang": "en", "dataset": "lex_files_500K-600K", "value": 0.45723}],
         task="qa",
         metric="ndcg_at_3",
         timestamp="2024-05-14T03:09:08Z",
@@ -45,11 +40,12 @@ def test_eval_result():
 
 
 @pytest.mark.parametrize(
-    'file_path',
+    "file_path",
     [
         "AIR-Bench_24.04/bge-m3/jina-reranker-v2-base-multilingual/results.json",
-        "AIR-Bench_24.05/bge-m3/NoReranker/results.json"
-    ])
+        "AIR-Bench_24.05/bge-m3/NoReranker/results.json",
+    ],
+)
 def test_full_eval_result_init_from_json_file(file_path):
     json_fp = cur_fp.parents[1] / "toydata/eval_results/" / file_path
     full_eval_result = FullEvalResult.init_from_json_file(json_fp)
@@ -59,20 +55,35 @@ def test_full_eval_result_init_from_json_file(file_path):
 
 
 @pytest.mark.parametrize(
-    'file_path, task, expected_num_results',
+    "file_path, task, expected_num_results",
     [
         ("AIR-Bench_24.04/bge-m3/jina-reranker-v2-base-multilingual/results.json", "qa", NUM_QA_BENCHMARKS_24_04),
-        ("AIR-Bench_24.04/bge-m3/jina-reranker-v2-base-multilingual/results.json", "long-doc", NUM_DOC_BENCHMARKS_24_04),
+        (
+            "AIR-Bench_24.04/bge-m3/jina-reranker-v2-base-multilingual/results.json",
+            "long-doc",
+            NUM_DOC_BENCHMARKS_24_04,
+        ),
         ("AIR-Bench_24.05/bge-m3/NoReranker/results.json", "qa", NUM_QA_BENCHMARKS_24_05),
         ("AIR-Bench_24.05/bge-m3/NoReranker/results.json", "long-doc", NUM_DOC_BENCHMARKS_24_05),
-    ])
+    ],
+)
 def test_full_eval_result_to_dict(file_path, task, expected_num_results):
     json_fp = cur_fp.parents[1] / "toydata/eval_results/" / file_path
     full_eval_result = FullEvalResult.init_from_json_file(json_fp)
     result_dict_list = full_eval_result.to_dict(task)
     assert len(result_dict_list) == 1
     result = result_dict_list[0]
-    attr_list = frozenset([
-        'eval_name', 'Retrieval Method', 'Reranking Model', 'Retrieval Model LINK', 'Reranking Model LINK', 'Revision', 'Submission Date', 'Anonymous Submission'])
+    attr_list = frozenset(
+        [
+            "eval_name",
+            "Retrieval Method",
+            "Reranking Model",
+            "Retrieval Model LINK",
+            "Reranking Model LINK",
+            "Revision",
+            "Submission Date",
+            "Anonymous Submission",
+        ]
+    )
     result_cols = list(result.keys())
-    assert len(result_cols) == (expected_num_results + len(attr_list))
+    assert len(result_cols) == (expected_num_results + len(attr_list))

tests/src/test_read_evals.py

@@ -1,78 +0,0 @@
-from pathlib import Path
-
-from src.models import FullEvalResult
-from src.read_evals import load_raw_eval_results
-from src.utils import get_leaderboard_df
-
-cur_fp = Path(__file__)
-
-
-def test_init_from_json_file():
-    json_fp = cur_fp.parents[2] / "toydata" / "test_data.json"
-    full_eval_result = FullEvalResult.init_from_json_file(json_fp)
-    num_different_task_domain_lang_metric_dataset_combination = 6
-    assert len(full_eval_result.results) == num_different_task_domain_lang_metric_dataset_combination
-    assert full_eval_result.retrieval_model == "bge-m3"
-    assert full_eval_result.reranking_model == "bge-reranker-v2-m3"
-
-
-def test_to_dict():
-    json_fp = cur_fp.parents[2] / "toydata" / "test_data.json"
-    full_eval_result = FullEvalResult.init_from_json_file(json_fp)
-    result_list = full_eval_result.to_dict(task="qa", metric="ndcg_at_1")
-    assert len(result_list) == 1
-    result_dict = result_list[0]
-    assert result_dict["Retrieval Model"] == "bge-m3"
-    assert result_dict["Reranking Model"] == "bge-reranker-v2-m3"
-    assert result_dict["wiki_en"] is not None
-    assert result_dict["wiki_zh"] is not None
-
-
-def test_get_raw_eval_results():
-    results_path = cur_fp.parents[2] / "toydata" / "eval_results" / "AIR-Bench_24.04"
-    results = load_raw_eval_results(results_path)
-    # only load the latest results
-    assert len(results) == 4
-    assert results[0].eval_name == "bge-base-en-v1.5_NoReranker"
-    assert len(results[0].results) == 70
-    assert results[0].eval_name == "bge-base-en-v1.5_bge-reranker-v2-m3"
-    assert len(results[1].results) == 70
-
-
-def test_get_leaderboard_df():
-    results_path = cur_fp.parents[2] / "toydata" / "eval_results" / "AIR-Bench_24.04"
-    raw_data = load_raw_eval_results(results_path)
-    df = get_leaderboard_df(raw_data, "qa", "ndcg_at_10")
-    assert df.shape[0] == 4
-    # the results contain only one embedding model
-    # for i in range(4):
-    #     assert df["Retrieval Model"][i] == "bge-m3"
-    # # the results contain only two reranking model
-    # assert df["Reranking Model"][0] == "bge-reranker-v2-m3"
-    # assert df["Reranking Model"][1] == "NoReranker"
-    # assert df["Average ⬆️"][0] > df["Average ⬆️"][1]
-    # assert not df[['Average ⬆️', 'wiki_en', 'wiki_zh', ]].isnull().values.any()
-
-
-def test_get_leaderboard_df_long_doc():
-    results_path = cur_fp.parents[2] / "toydata" / "test_results"
-    raw_data = load_raw_eval_results(results_path)
-    df = get_leaderboard_df(raw_data, "long-doc", "ndcg_at_1")
-    assert df.shape[0] == 2
-    # the results contain only one embedding model
-    for i in range(2):
-        assert df["Retrieval Model"][i] == "bge-m3"
-    # the results contains only two reranking model
-    assert df["Reranking Model"][0] == "bge-reranker-v2-m3"
-    assert df["Reranking Model"][1] == "NoReranker"
-    assert df["Average ⬆️"][0] > df["Average ⬆️"][1]
-    assert (
-        not df[
-            [
-                "Average ⬆️",
-                "law_en_lex_files_500k_600k",
-            ]
-        ]
-        .isnull()
-        .values.any()
-    )

tests/src/test_utils.py

@@ -1,10 +1,21 @@
-import pytest
-import pandas as pd
 from pathlib import Path
 
-from src.utils import remove_html, calculate_mean, filter_models, filter_queries, get_default_cols, select_columns, get_selected_cols, _update_df_elem, get_leaderboard_df
-from src.models import model_hyperlink, TaskType
+import pandas as pd
+import pytest
+
 from src.columns import COL_NAME_RERANKING_MODEL, COL_NAME_RETRIEVAL_MODEL
+from src.models import TaskType, model_hyperlink
+from src.utils import (
+    _update_df_elem,
+    calculate_mean,
+    filter_models,
+    filter_queries,
+    get_default_cols,
+    get_leaderboard_df,
+    get_selected_cols,
+    remove_html,
+    select_columns,
+)
 
 cur_fp = Path(__file__)
 
@@ -18,18 +29,8 @@ NUM_DOC_BENCHMARKS_24_04 = 15
 def toy_df():
     return pd.DataFrame(
         {
-            "Retrieval Method": [
-                "bge-m3",
-                "bge-m3",
-                "jina-embeddings-v2-base",
-                "jina-embeddings-v2-base"
-            ],
-            "Reranking Model": [
-                "bge-reranker-v2-m3",
-                "NoReranker",
-                "bge-reranker-v2-m3",
-                "NoReranker"
-            ],
+            "Retrieval Method": ["bge-m3", "bge-m3", "jina-embeddings-v2-base", "jina-embeddings-v2-base"],
+            "Reranking Model": ["bge-reranker-v2-m3", "NoReranker", "bge-reranker-v2-m3", "NoReranker"],
             "Rank 🏆": [1, 2, 3, 4],
             "Revision": ["123", "234", "345", "456"],
             "Submission Date": ["", "", "", ""],
@@ -45,8 +46,7 @@ def toy_df():
 
 def test_remove_html():
     model_name = "jina-embeddings-v3"
-    html_str = model_hyperlink(
-        "https://jina.ai", model_name)
+    html_str = model_hyperlink("https://jina.ai", model_name)
     output_str = remove_html(html_str)
     assert output_str == model_name
 
@@ -60,17 +60,29 @@ def test_calculate_mean():
     assert result[1] == -1
 
 
-@pytest.mark.parametrize("models, expected", [
-    (["model1", "model3"], 2),
-    (["model1", "model_missing"], 1),
-    (["model1", "model2", "model3"], 3),
-    (["model1", ], 1),
-    ([], 3),
-])
+@pytest.mark.parametrize(
+    "models, expected",
+    [
+        (["model1", "model3"], 2),
+        (["model1", "model_missing"], 1),
+        (["model1", "model2", "model3"], 3),
+        (
+            [
+                "model1",
+            ],
+            1,
+        ),
+        ([], 3),
+    ],
+)
 def test_filter_models(models, expected):
     df = pd.DataFrame(
         {
-            COL_NAME_RERANKING_MODEL: ["model1", "model2", "model3", ],
+            COL_NAME_RERANKING_MODEL: [
+                "model1",
+                "model2",
+                "model3",
+            ],
             "col2": [1, 2, 3],
         }
     )
@@ -78,18 +90,29 @@ def test_filter_models(models, expected):
     assert len(output_df) == expected
 
 
-@pytest.mark.parametrize("query, expected", [
-    ("model1;model3", 2),
-    ("model1;model4", 1),
-    ("model1;model2;model3", 3),
-    ("model1", 1),
-    ("", 3),
-])
+@pytest.mark.parametrize(
+    "query, expected",
+    [
+        ("model1;model3", 2),
+        ("model1;model4", 1),
+        ("model1;model2;model3", 3),
+        ("model1", 1),
+        ("", 3),
+    ],
+)
 def test_filter_queries(query, expected):
     df = pd.DataFrame(
         {
-            COL_NAME_RETRIEVAL_MODEL: ["model1", "model2", "model3", ],
-            COL_NAME_RERANKING_MODEL: ["model4", "model5", "model6", ],
+            COL_NAME_RETRIEVAL_MODEL: [
+                "model1",
+                "model2",
+                "model3",
+            ],
+            COL_NAME_RERANKING_MODEL: [
+                "model4",
+                "model5",
+                "model6",
+            ],
         }
     )
     output_df = filter_queries(query, df)
@@ -103,10 +126,10 @@ def test_filter_queries(query, expected):
         (TaskType.long_doc, "air_bench_2404", True, NUM_DOC_BENCHMARKS_24_04),
         (TaskType.qa, "air_bench_2405", False, NUM_QA_BENCHMARKS_24_05),
         (TaskType.long_doc, "air_bench_2405", False, NUM_DOC_BENCHMARKS_24_05),
-    ]
+    ],
 )
 def test_get_default_cols(task_type, slug, add_fix_cols, expected):
-    attr_cols = ['Rank 🏆', 'Retrieval Method', 'Reranking Model', 'Revision', 'Submission Date', 'Average ⬆️']
+    attr_cols = ["Rank 🏆", "Retrieval Method", "Reranking Model", "Revision", "Submission Date", "Average ⬆️"]
     cols, types = get_default_cols(task_type, slug)
     cols_set = frozenset(cols)
     attrs_set = frozenset(attr_cols)
@@ -119,44 +142,54 @@ def test_get_default_cols(task_type, slug, add_fix_cols, expected):
 @pytest.mark.parametrize(
     "task_type, domains, languages, expected",
     [
-        (TaskType.qa, ["wiki", "news"], ["zh",], ["wiki_zh", "news_zh"]),
-        (TaskType.qa, ["law",], ["zh", "en"], ["law_en"]),
+        (
+            TaskType.qa,
+            ["wiki", "news"],
+            [
+                "zh",
+            ],
+            ["wiki_zh", "news_zh"],
+        ),
+        (
+            TaskType.qa,
+            [
+                "law",
+            ],
+            ["zh", "en"],
+            ["law_en"],
+        ),
         (
             TaskType.long_doc,
             ["healthcare"],
             ["zh", "en"],
             [
-                'healthcare_en_pubmed_100k_200k_1',
-                'healthcare_en_pubmed_100k_200k_2',
-                'healthcare_en_pubmed_100k_200k_3',
-                'healthcare_en_pubmed_40k_50k_5_merged',
-                'healthcare_en_pubmed_30k_40k_10_merged'
-            ]
-        )
-    ]
+                "healthcare_en_pubmed_100k_200k_1",
+                "healthcare_en_pubmed_100k_200k_2",
+                "healthcare_en_pubmed_100k_200k_3",
+                "healthcare_en_pubmed_40k_50k_5_merged",
+                "healthcare_en_pubmed_30k_40k_10_merged",
+            ],
+        ),
+    ],
 )
 def test_get_selected_cols(task_type, domains, languages, expected):
     slug = "air_bench_2404"
     cols = get_selected_cols(task_type, slug, domains, languages)
     assert sorted(cols) == sorted(expected)
 
+
 @pytest.mark.parametrize("reset_rank", [False])
 def test_select_columns(toy_df, reset_rank):
     expected = [
-        'Rank 🏆',
-        'Retrieval Method',
-        'Reranking Model',
-        'Revision',
-        'Submission Date',
-        'Average ⬆️',
-        'news_zh']
-    df_result = select_columns(
-        toy_df,
-        ["news"],
-        ["zh"],
-        version_slug="air_bench_2404",
-        reset_ranking=reset_rank
-    )
+        "Rank 🏆",
+        "Retrieval Method",
+        "Reranking Model",
+        "Revision",
+        "Submission Date",
+        "Average ⬆️",
+        "news_zh",
+    ]
+    df_result = select_columns(toy_df, ["news"], ["zh"], version_slug="air_bench_2404", reset_ranking=reset_rank)
     assert len(df_result.columns) == len(expected)
     if reset_rank:
         assert df_result["Average ⬆️"].equals(df_result["news_zh"])
@@ -170,20 +203,10 @@ def test_select_columns(toy_df, reset_rank):
         (False, True),
         (True, True),
         (True, False),
-    ]
+    ],
 )
 def test__update_df_elem(toy_df, reset_rank, show_anony):
-    df = _update_df_elem(
-        TaskType.qa,
-        "AIR-Bench_24.04",
-        toy_df,
-        ["news"],
-        ["zh"],
-        [],
-        "",
-        show_anony,
-        reset_rank
-    )
+    df = _update_df_elem(TaskType.qa, "AIR-Bench_24.04", toy_df, ["news"], ["zh"], [], "", show_anony, reset_rank)
     if show_anony:
         assert df.shape[0] == 4
     else:
@@ -201,19 +224,14 @@ def test__update_df_elem(toy_df, reset_rank, show_anony):
         ("AIR-Bench_24.04", TaskType.qa),
         ("AIR-Bench_24.04", TaskType.long_doc),
         ("AIR-Bench_24.05", TaskType.qa),
-        ("AIR-Bench_24.05", TaskType.long_doc)
-    ]
+        ("AIR-Bench_24.05", TaskType.long_doc),
+    ],
 )
 def test_get_leaderboard_df(version, task_type):
     from src.loaders import load_raw_eval_results
     from src.models import LeaderboardDataStore, get_safe_name
-    raw_data = load_raw_eval_results(
-        cur_fp.parents[1] / f"toydata/eval_results/{version}"
-    )
+
+    raw_data = load_raw_eval_results(cur_fp.parents[1] / f"toydata/eval_results/{version}")
     ds = LeaderboardDataStore(version, get_safe_name(version), raw_data=raw_data)
-    df = get_leaderboard_df(
-        ds,
-        task_type,
-        "ndcg_at_10"
-    )
-    assert df.shape[0] == 1
+    df = get_leaderboard_df(ds, task_type, "ndcg_at_10")
+    assert df.shape[0] == 1

tests/test_utils.py

@@ -1,136 +0,0 @@
-import pandas as pd
-import pytest
-
-from app import update_table
-from src.columns import (
-    COL_NAME_AVG,
-    COL_NAME_IS_ANONYMOUS,
-    COL_NAME_RANK,
-    COL_NAME_RERANKING_MODEL,
-    COL_NAME_RETRIEVAL_MODEL,
-    COL_NAME_REVISION,
-    COL_NAME_TIMESTAMP,
-)
-from src.utils import (
-    filter_models,
-    filter_queries,
-    get_default_cols,
-    get_iso_format_timestamp,
-    search_table,
-    select_columns,
-    update_doc_df_elem,
-)
-
-
-@pytest.fixture
-def toy_df():
-    return pd.DataFrame(
-        {
-            "Retrieval Model": ["bge-m3", "bge-m3", "jina-embeddings-v2-base", "jina-embeddings-v2-base"],
-            "Reranking Model": ["bge-reranker-v2-m3", "NoReranker", "bge-reranker-v2-m3", "NoReranker"],
-            "Average ⬆️": [0.6, 0.4, 0.3, 0.2],
-            "wiki_en": [0.8, 0.7, 0.2, 0.1],
-            "wiki_zh": [0.4, 0.1, 0.4, 0.3],
-            "news_en": [0.8, 0.7, 0.2, 0.1],
-            "news_zh": [0.4, 0.1, 0.4, 0.3],
-        }
-    )
-
-
-@pytest.fixture
-def toy_df_long_doc():
-    return pd.DataFrame(
-        {
-            "Retrieval Model": ["bge-m3", "bge-m3", "jina-embeddings-v2-base", "jina-embeddings-v2-base"],
-            "Reranking Model": ["bge-reranker-v2-m3", "NoReranker", "bge-reranker-v2-m3", "NoReranker"],
-            "Average ⬆️": [0.6, 0.4, 0.3, 0.2],
-            "law_en_lex_files_300k_400k": [0.4, 0.1, 0.4, 0.3],
-            "law_en_lex_files_400k_500k": [0.8, 0.7, 0.2, 0.1],
-            "law_en_lex_files_500k_600k": [0.8, 0.7, 0.2, 0.1],
-            "law_en_lex_files_600k_700k": [0.4, 0.1, 0.4, 0.3],
-        }
-    )
-
-
-def test_filter_models(toy_df):
-    df_result = filter_models(
-        toy_df,
-        [
-            "bge-reranker-v2-m3",
-        ],
-    )
-    assert len(df_result) == 2
-    assert df_result.iloc[0]["Reranking Model"] == "bge-reranker-v2-m3"
-
-
-def test_search_table(toy_df):
-    df_result = search_table(toy_df, "jina")
-    assert len(df_result) == 2
-    assert df_result.iloc[0]["Retrieval Model"] == "jina-embeddings-v2-base"
-
-
-def test_filter_queries(toy_df):
-    df_result = filter_queries("jina", toy_df)
-    assert len(df_result) == 2
-    assert df_result.iloc[0]["Retrieval Model"] == "jina-embeddings-v2-base"
-
-
-
-
-
-def test_update_table_long_doc(toy_df_long_doc):
-    df_result = update_doc_df_elem(
-        toy_df_long_doc,
-        [
-            "law",
-        ],
-        [
-            "en",
-        ],
-        [
-            "bge-reranker-v2-m3",
-        ],
-        "jina",
-    )
-    print(df_result)
-
-
-def test_get_iso_format_timestamp():
-    timestamp_config, timestamp_fn = get_iso_format_timestamp()
-    assert len(timestamp_fn) == 14
-    assert len(timestamp_config) == 20
-    assert timestamp_config[-1] == "Z"
-
-
-def test_get_default_cols():
-    cols, types = get_default_cols("qa")
-    for c, t in zip(cols, types):
-        print(f"type({c}): {t}")
-    assert len(frozenset(cols)) == len(cols)
-
-
-def test_update_table():
-    df = pd.DataFrame(
-        {
-            COL_NAME_IS_ANONYMOUS: [False, False, False],
-            COL_NAME_REVISION: ["a1", "a2", "a3"],
-            COL_NAME_TIMESTAMP: ["2024-05-12T12:24:02Z"] * 3,
-            COL_NAME_RERANKING_MODEL: ["NoReranker"] * 3,
-            COL_NAME_RETRIEVAL_MODEL: ["Foo"] * 3,
-            COL_NAME_RANK: [1, 2, 3],
-            COL_NAME_AVG: [0.1, 0.2, 0.3],  # unsorted values
-            "wiki_en": [0.1, 0.2, 0.3],
-        }
-    )
-    results = update_table(
-        df,
-        "wiki",
-        "en",
-        ["NoReranker"],
-        "",
-        show_anonymous=False,
-        reset_ranking=False,
-        show_revision_and_timestamp=False,
-    )
-    # keep the RANK as the same regardless of the unsorted averages
-    assert results[COL_NAME_RANK].to_list() == [1, 2, 3]

tests/toydata/test_data.json

@@ -1,98 +0,0 @@
-[
-  {
-    "config": {
-      "retrieval_model": "bge-m3",
-      "reranking_model": "bge-reranker-v2-m3",
-      "task": "long_doc",
-      "metric": "ndcg_at_1"
-    },
-    "results": [
-      {
-        "domain": "law",
-        "lang": "en",
-        "dataset": "lex_files_500K-600K",
-        "value": 0.75723
-      }
-    ]
-  },
-  {
-    "config": {
-      "retrieval_model": "bge-m3",
-      "reranking_model": "bge-reranker-v2-m3",
-      "task": "long_doc",
-      "metric": "ndcg_at_3"
-    },
-    "results": [
-      {
-        "domain": "law",
-        "lang": "en",
-        "dataset": "lex_files_500K-600K",
-        "value": 0.69909
-      }
-    ]
-  },
-  {
-    "config": {
-      "retrieval_model": "bge-m3",
-      "reranking_model": "bge-reranker-v2-m3",
-      "task": "qa",
-      "metric": "ndcg_at_1"
-    },
-    "results": [
-      {
-        "domain": "wiki",
-        "lang": "en",
-        "dataset": "unknown",
-        "value": 0.69083
-      }
-    ]
-  },
-  {
-    "config": {
-      "retrieval_model": "bge-m3",
-      "reranking_model": "bge-reranker-v2-m3",
-      "task": "qa",
-      "metric": "ndcg_at_3"
-    },
-    "results": [
-      {
-        "domain": "wiki",
-        "lang": "en",
-        "dataset": "unknown",
-        "value": 0.73359
-      }
-    ]
-  },
-  {
-    "config": {
-      "retrieval_model": "bge-m3",
-      "reranking_model": "bge-reranker-v2-m3",
-      "task": "qa",
-      "metric": "ndcg_at_1"
-    },
-    "results": [
-      {
-        "domain": "wiki",
-        "lang": "zh",
-        "dataset": "unknown",
-        "value": 0.78358
-      }
-    ]
-  },
-  {
-    "config": {
-      "retrieval_model": "bge-m3",
-      "reranking_model": "bge-reranker-v2-m3",
-      "task": "qa",
-      "metric": "ndcg_at_3"
-    },
-    "results": [
-      {
-        "domain": "wiki",
-        "lang": "zh",
-        "dataset": "unknown",
-        "value": 0.78358
-      }
-    ]
-  }
-]

tests/toydata/test_results/bge-m3/NoReranker/results_2023-11-21T18-10-08.json

@@ -1,98 +0,0 @@
-[
-  {
-    "config": {
-      "retrieval_model": "bge-m3",
-      "reranking_model": "NoReranker",
-      "task": "long_doc",
-      "metric": "ndcg_at_1"
-    },
-    "results": [
-      {
-        "domain": "law",
-        "lang": "en",
-        "dataset": "lex_files_500K-600K",
-        "value": 0.45723
-      }
-    ]
-  },
-  {
-    "config": {
-      "retrieval_model": "bge-m3",
-      "reranking_model": "NoReranker",
-      "task": "long_doc",
-      "metric": "ndcg_at_3"
-    },
-    "results": [
-      {
-        "domain": "law",
-        "lang": "en",
-        "dataset": "lex_files_500K-600K",
-        "value": 0.49909
-      }
-    ]
-  },
-  {
-    "config": {
-      "retrieval_model": "bge-m3",
-      "reranking_model": "NoReranker",
-      "task": "qa",
-      "metric": "ndcg_at_1"
-    },
-    "results": [
-      {
-        "domain": "wiki",
-        "lang": "en",
-        "dataset": "unknown",
-        "value": 0.49083
-      }
-    ]
-  },
-  {
-    "config": {
-      "retrieval_model": "bge-m3",
-      "reranking_model": "NoReranker",
-      "task": "qa",
-      "metric": "ndcg_at_3"
-    },
-    "results": [
-      {
-        "domain": "wiki",
-        "lang": "en",
-        "dataset": "unknown",
-        "value": 0.43359
-      }
-    ]
-  },
-  {
-    "config": {
-      "retrieval_model": "bge-m3",
-      "reranking_model": "NoReranker",
-      "task": "qa",
-      "metric": "ndcg_at_1"
-    },
-    "results": [
-      {
-        "domain": "wiki",
-        "lang": "zh",
-        "dataset": "unknown",
-        "value": 0.78358
-      }
-    ]
-  },
-  {
-    "config": {
-      "retrieval_model": "bge-m3",
-      "reranking_model": "NoReranker",
-      "task": "qa",
-      "metric": "ndcg_at_3"
-    },
-    "results": [
-      {
-        "domain": "wiki",
-        "lang": "zh",
-        "dataset": "unknown",
-        "value": 0.78358
-      }
-    ]
-  }
-]

tests/toydata/test_results/bge-m3/bge-reranker-v2-m3/results_2023-11-21T18-10-08.json

@@ -1,98 +0,0 @@
-[
-  {
-    "config": {
-      "retrieval_model": "bge-m3",
-      "reranking_model": "bge-reranker-v2-m3",
-      "task": "long_doc",
-      "metric": "ndcg_at_1"
-    },
-    "results": [
-      {
-        "domain": "law",
-        "lang": "en",
-        "dataset": "lex_files_500K-600K",
-        "value": 0.75723
-      }
-    ]
-  },
-  {
-    "config": {
-      "retrieval_model": "bge-m3",
-      "reranking_model": "bge-reranker-v2-m3",
-      "task": "long_doc",
-      "metric": "ndcg_at_3"
-    },
-    "results": [
-      {
-        "domain": "law",
-        "lang": "en",
-        "dataset": "lex_files_500K-600K",
-        "value": 0.69909
-      }
-    ]
-  },
-  {
-    "config": {
-      "retrieval_model": "bge-m3",
-      "reranking_model": "bge-reranker-v2-m3",
-      "task": "qa",
-      "metric": "ndcg_at_1"
-    },
-    "results": [
-      {
-        "domain": "wiki",
-        "lang": "en",
-        "dataset": "unknown",
-        "value": 0.69083
-      }
-    ]
-  },
-  {
-    "config": {
-      "retrieval_model": "bge-m3",
-      "reranking_model": "bge-reranker-v2-m3",
-      "task": "qa",
-      "metric": "ndcg_at_3"
-    },
-    "results": [
-      {
-        "domain": "wiki",
-        "lang": "en",
-        "dataset": "unknown",
-        "value": 0.73359
-      }
-    ]
-  },
-  {
-    "config": {
-      "retrieval_model": "bge-m3",
-      "reranking_model": "bge-reranker-v2-m3",
-      "task": "qa",
-      "metric": "ndcg_at_1"
-    },
-    "results": [
-      {
-        "domain": "wiki",
-        "lang": "zh",
-        "dataset": "unknown",
-        "value": 0.78358
-      }
-    ]
-  },
-  {
-    "config": {
-      "retrieval_model": "bge-m3",
-      "reranking_model": "bge-reranker-v2-m3",
-      "task": "qa",
-      "metric": "ndcg_at_3"
-    },
-    "results": [
-      {
-        "domain": "wiki",
-        "lang": "zh",
-        "dataset": "unknown",
-        "value": 0.78358
-      }
-    ]
-  }
-]