feat: seperate the qa and longdoc tasks

src/benchmarks.py

@@ -110,7 +110,8 @@ class Benchmark:
     metric: str  # ndcg_at_1 ,metric_key in the json file
     col_name: str  # [domain]_[language], name to display in the leaderboard
 
-benchmark_dict = {}
+qa_benchmark_dict = {}
+long_doc_benchmark_dict = {}
 for task, domain_dict in dataset_dict.items():
     for domain, lang_dict in domain_dict.items():
         for lang, dataset_list in lang_dict.items():
@@ -119,13 +120,14 @@ for task, domain_dict in dataset_dict.items():
                 benchmark_name = get_safe_name(benchmark_name)
                 col_name = f"{domain}_{lang}"
                 for metric in dataset_list:
-                    benchmark_dict[benchmark_name] = Benchmark(benchmark_name, metric, col_name)
+                    qa_benchmark_dict[benchmark_name] = Benchmark(benchmark_name, metric, col_name)
             elif task == "long_doc":
                 for dataset in dataset_list:
                     col_name = f"{domain}_{lang}_{dataset}"
                     for metric in metric_list:
                         benchmark_name = f"{task}_{domain}_{lang}_{dataset}_{metric}"
                         benchmark_name = get_safe_name(benchmark_name)
-                        benchmark_dict[benchmark_name] = Benchmark(benchmark_name, metric, col_name)
+                        long_doc_benchmark_dict[benchmark_name] = Benchmark(benchmark_name, metric, col_name)
 
-Benchmarks = Enum('Benchmarks', benchmark_dict)
+BenchmarksQA = Enum('BenchmarksQA', qa_benchmark_dict)
+BenchmarksLongDoc = Enum('BenchmarksLongDoc', long_doc_benchmark_dict)

src/display/utils.py

@@ -1,6 +1,6 @@
 from dataclasses import dataclass, make_dataclass
 
-from src.benchmarks import Benchmarks
+from src.benchmarks import BenchmarksQA, BenchmarksLongDoc
 
 
 def fields(raw_class):
@@ -19,25 +19,32 @@ class ColumnContent:
     never_hidden: bool = False
 
 
-## Leaderboard columns
-auto_eval_column_dict = []
-# Init
-auto_eval_column_dict.append(
-    ["retrieval_model", ColumnContent, ColumnContent("Retrieval Model", "markdown", True, never_hidden=True)]
-)
-auto_eval_column_dict.append(
-    ["reranking_model", ColumnContent, ColumnContent("Reranking Model", "markdown", True, never_hidden=True)]
-)
-auto_eval_column_dict.append(
-    ["average", ColumnContent, ColumnContent("Average ⬆️", "number", True)]
-)
-for benchmark in Benchmarks:
+def make_autoevalcolumn(cls_name="BenchmarksQA", benchmarks=BenchmarksQA):
+    ## Leaderboard columns
+    auto_eval_column_dict = []
+    # Init
     auto_eval_column_dict.append(
-        [benchmark.name, ColumnContent, ColumnContent(benchmark.value.col_name, "number", True)]
+        ["retrieval_model", ColumnContent, ColumnContent("Retrieval Model", "markdown", True, never_hidden=True)]
     )
+    auto_eval_column_dict.append(
+        ["reranking_model", ColumnContent, ColumnContent("Reranking Model", "markdown", True, never_hidden=True)]
+    )
+    auto_eval_column_dict.append(
+        ["average", ColumnContent, ColumnContent("Average ⬆️", "number", True)]
+    )
+    for benchmark in benchmarks:
+        auto_eval_column_dict.append(
+            [benchmark.name, ColumnContent, ColumnContent(benchmark.value.col_name, "number", True)]
+        )
 
-# We use make dataclass to dynamically fill the scores from Tasks
-AutoEvalColumn = make_dataclass("AutoEvalColumn", auto_eval_column_dict, frozen=True)
+    # We use make dataclass to dynamically fill the scores from Tasks
+    return make_dataclass(cls_name, auto_eval_column_dict, frozen=True)
+
+
+AutoEvalColumnQA = make_autoevalcolumn(
+    "AutoEvalColumnQA", BenchmarksQA)
+AutoEvalColumnLongDoc = make_autoevalcolumn(
+    "AutoEvalColumnLongDoc", BenchmarksLongDoc)
 
 
 ## For the queue columns in the submission tab
@@ -48,10 +55,12 @@ class EvalQueueColumn:  # Queue column
 
 
 # Column selection
-COLS = [c.name for c in fields(AutoEvalColumn) if not c.hidden]
-TYPES = [c.type for c in fields(AutoEvalColumn) if not c.hidden]
-COLS_LITE = [c.name for c in fields(AutoEvalColumn) if c.displayed_by_default and not c.hidden]
+COLS = [c.name for c in fields(AutoEvalColumnQA) if not c.hidden]
+TYPES = [c.type for c in fields(AutoEvalColumnQA) if not c.hidden]
+COLS_LITE = [c.name for c in fields(AutoEvalColumnQA) if c.displayed_by_default and not c.hidden]
 
 EVAL_COLS = [c.name for c in fields(EvalQueueColumn)]
 
-BENCHMARK_COLS = [t.value.col_name for t in Benchmarks]
+QA_BENCHMARK_COLS = [t.value.col_name for t in BenchmarksQA]
+
+LONG_DOC_BENCHMARK_COLS = [t.value.col_name for t in BenchmarksLongDoc]

tests/src/display/test_utils.py

@@ -1,10 +1,11 @@
 import pytest
-from src.display.utils import fields, AutoEvalColumn, COLS, COLS_LITE, TYPES, EVAL_COLS, BENCHMARK_COLS
+from src.display.utils import fields, AutoEvalColumnQA, AutoEvalColumnLongDoc, COLS, COLS_LITE, TYPES, EVAL_COLS, QA_BENCHMARK_COLS, LONG_DOC_BENCHMARK_COLS
 
 
+@pytest.mark.parametrize('auto_eval_column')
 def test_fields():
-    for c in fields(AutoEvalColumn):
-        print(c.name)
+    for c in fields(AutoEvalColumnQA):
+        print(c)
 
 
 def test_macro_variables():
@@ -12,4 +13,4 @@ def test_macro_variables():
     print(f'COLS_LITE: {COLS_LITE}')
     print(f'TYPES: {TYPES}')
     print(f'EVAL_COLS: {EVAL_COLS}')
-    print(f'BENCHMARK_COLS: {BENCHMARK_COLS}')
+    print(f'BENCHMARK_COLS: {QA_BENCHMARK_COLS}')