update

halueval-cli.py

@@ -28,20 +28,22 @@ def main():
     eval_requests: list[EvalRequest] = get_eval_requests(job_status=status, hf_repo=QUEUE_REPO, local_dir=EVAL_REQUESTS_PATH_BACKEND)
     eval_request = [r for r in eval_requests if 'bloom-560m' in r.model][0]
 
-    task_names = ['halueval_qa']
+    # task_names = ['halueval_qa']
+    # task_names = ['triviaqa']
+    TASKS_HARNESS = [task.value for task in Tasks]
 
     include_task_folder("src/backend/tasks/")
     initialize_tasks('INFO')
 
     print(tasks.ALL_TASKS)
 
-    task_names = utils.pattern_match(task_names, tasks.ALL_TASKS)
-
-    for task_name in task_names:
-        print(f"Selected Tasks: [{task_name}]")
-        results = evaluator.simple_evaluate(model="hf-auto", model_args=eval_request.get_model_args(), tasks=[task_name], num_fewshot=0,
+    for task in TASKS_HARNESS:
+        print(f"Selected Tasks: [{task}]")
+        results = evaluator.simple_evaluate(model="hf", model_args=eval_request.get_model_args(), tasks=[task.benchmark], num_fewshot=0,
                                             batch_size=1, device=DEVICE, use_cache=None, limit=8, write_out=True)
-        print('AAA', results)
+        print('AAA', results["results"])
+
+        # breakpoint()
 
 
 if __name__ == "__main__":

src/backend/tasks/halueval/halueval_dialogue.yaml

@@ -4,23 +4,11 @@ dataset_name: dialogue_samples
 output_type: generate_until
 training_split: data
 validation_split: data
+test_split: data
+num_fewshot: 0
 doc_to_text: !function utils.doc_to_text_dialogue
 doc_to_target: !function utils.doc_to_target_qa
 process_results: !function utils.process_results_qa
-fewshot_delimiter: "\n"
-generation_kwargs:
-  until:
-    - "\n"
-    - "."
-    - ","
-  do_sample: false
-  temperature: 0.0
-filter_list:
-  - name: remove_whitespace
-    filter:
-      - function: remove_whitespace
-      - function: take_first
-target_delimiter: " "
 metric_list:
   - metric: em
     aggregation: mean

src/backend/tasks/halueval/halueval_qa.yaml

@@ -4,23 +4,11 @@ dataset_name: qa_samples
 output_type: generate_until
 training_split: data
 validation_split: data
+test_split: data
+num_fewshot: 0
 doc_to_text: !function utils.doc_to_text_qa
 doc_to_target: !function utils.doc_to_target_qa
 process_results: !function utils.process_results_qa
-fewshot_delimiter: "\n"
-generation_kwargs:
-  until:
-    - "\n"
-    - "."
-    - ","
-  do_sample: false
-  temperature: 0.0
-filter_list:
-  - name: remove_whitespace
-    filter:
-      - function: remove_whitespace
-      - function: take_first
-target_delimiter: " "
 metric_list:
   - metric: em
     aggregation: mean

src/backend/tasks/halueval/halueval_summarization.yaml

@@ -4,23 +4,11 @@ dataset_name: summarization_samples
 output_type: generate_until
 training_split: data
 validation_split: data
+test_split: data
+num_fewshot: 0
 doc_to_text: !function utils.doc_to_text_summarization
 doc_to_target: !function utils.doc_to_target_qa
 process_results: !function utils.process_results_qa
-fewshot_delimiter: "\n"
-generation_kwargs:
-  until:
-    - "\n"
-    - "."
-    - ","
-  do_sample: false
-  temperature: 0.0
-filter_list:
-  - name: remove_whitespace
-    filter:
-      - function: remove_whitespace
-      - function: take_first
-target_delimiter: " "
 metric_list:
   - metric: em
     aggregation: mean

src/backend/tasks/halueval/utils.py

@@ -116,7 +116,7 @@ def compute_metrics_qa(gold_answer: str, prediction: str) -> dict[str, float]:
     elif "No" in prediction:
         prediction = "no"
 
-    is_exact = (gold_answer == prediction)
+    is_exact = gold_answer == prediction
 
     res = {"correctness": 1.0 if is_correct else 0.0}
     if is_correct:

src/leaderboard/read_evals.py

@@ -86,7 +86,17 @@ class EvalResult:
                     continue
 
             # We average all scores of a given metric (mostly for mmlu)
-            accs = np.array([v.get(task.metric, None) for k, v in data["results"].items() if task.benchmark in k])
+
+            def post_process_results(results: dict) -> dict:
+                res_copy = results.copy()
+                for k, v in res_copy.items():
+                    if "," in k:
+                        tokens = k.split(",")
+                        results[tokens[0]] = v
+                return results
+
+            accs = np.array([v.get(task.metric, None) for k, v in post_process_results(data["results"]).items() if task.benchmark in k])
+
             if accs.size == 0 or any([acc is None for acc in accs]):
                 continue