update

lm_eval/tasks/halueval/halueval_qa.yaml

@@ -0,0 +1,32 @@
+task: halueval_qa
+dataset_path: pminervini/HaluEval
+dataset_name: qa_samples
+output_type: generate_until
+training_split: data
+validation_split: data
+doc_to_text: !function utils.doc_to_text_qa
+doc_to_target: !function utils.doc_to_target_qa
+process_results: !function utils.process_results_qa
+fewshot_delimiter: "\n"
+generation_kwargs:
+  until:
+    - "\n"
+    - "."
+    - ","
+  do_sample: false
+  temperature: 0.0
+filter_list:
+  - name: remove_whitespace
+    filter:
+      - function: remove_whitespace
+      - function: take_first
+target_delimiter: " "
+metric_list:
+  - metric: em
+    aggregation: mean
+    higher_is_better: true
+  - metric: f1
+    aggregation: mean
+    higher_is_better: true
+metadata:
+  - version: 0.0

lm_eval/tasks/halueval/utils.py

@@ -0,0 +1,87 @@
+from itertools import zip_longest
+import transformers.data.metrics.squad_metrics as squad_metrics
+
+QA_INSTURCTIONS = """I want you act as an answer judge. Given a question and an answer, your objective is to determine if the provided answer contains non-factual or hallucinated information. You SHOULD give your judgement based on the following hallucination types and the world knowledge.
+
+You are trying to determine if the answer misunderstands the question context and intention.
+#Question#: What is a rare breed of dog that was derived as a variant of Rat Terrier, Shiloh Shepherd dog or American Hairless Terrier?
+#Answer#: American Hairless Terrier
+#Your Judgement#: No
+
+You are trying to determine if there is a factual contradiction between the answer and the world knowledge. Some information in the answer might be fabricated.
+#Question#: Are the New Orleans Outfall Canals the same length as the Augusta Canal?
+#Answer#: No, the New Orleans Outfall Canals and the Augusta Canal are not the same length. The Orleans Canal is approximately 3.6 miles (5.8 kilometers) long while the Augusta Canal is approximately 7 miles (11.3 kilometers) long.
+#Your Judgement#: Yes
+#Question#: What U.S Highway gives access to Zilpo Road, and is also known as Midland Trail?
+#Answer#: U.S Highway 70
+#Your Judgement#: Yes
+
+You are trying to determine if the answer is too general or too specific to answer the question at an appropriate level of specificity.
+#Question#: What genre do Superheaven and Oceansize belong to?
+#Answer#: Superheaven and Oceansize belong to the rock genre.
+#Your Judgement#: No
+#Question#: What profession do Kōbō Abe and Agatha Christie share?
+#Answer#: Playwright.
+#Your Judgement#: No
+
+You are trying to determine if the answer can be correctly inferred from the knowledge.
+#Question#: Which band has more members, Muse or The Raconteurs?
+#Answer#: Muse has more members than The Raconteurs.
+#Your Judgement#: Yes
+#Question#: Which is currently more valuable, Temagami-Lorrain Mine or Meadowbank Gold Mine?
+#Answer#: Meadowbank Gold Mine, since Meadowbank Gold Mine is still producing gold and the TemagamiLorrain Mine has been inactive for years.
+#Your Judgement#: No
+
+You should try your best to determine if the answer contains non-factual or hallucinated information according to the above hallucination types. The answer you give MUST be \"Yes\" or \"No\""."""
+
+
+def doc_to_text_qa(doc: dict[str, str]) -> str:
+    doc_text = QA_INSTURCTIONS + "\n\n#Question#: " + doc["question"] + "\n#Answer#: " + doc["answer"] + "\n#Your Judgement#:"
+    return doc_text
+
+
+def doc_to_target_qa(doc: dict[str, str]) -> str:
+    return doc['hallucination']
+
+
+def em(gold_list: list[str], predictions: list[str]):
+    # tests for exact match and on the normalised answer (compute_exact)
+    em_sum = 0.0
+    if len(gold_list) > 1:
+        for i in range(len(gold_list)):
+            gold_answers = gold_list[0:i] + gold_list[i + 1 :]
+            # predictions compared against (n) golds and take maximum
+            em_sum += max(squad_metrics.compute_exact(a, predictions) for a in gold_answers)
+    else:
+        em_sum += max(squad_metrics.compute_exact(a, predictions) for a in gold_list)
+    return em_sum / max(1, len(gold_list))
+
+
+def compute_metrics(gold_list: list[str], predictions: list[str]) -> dict[str, float]:
+    f1_sum = 0.0
+    em_sum = 0.0
+
+    is_correct_lst = []
+    is_exact_lst = []
+
+    if len(gold_list) > 1:
+        for i in range(len(gold_list)):
+            gold_answers = gold_list[0:i] + gold_list[i + 1 :]
+            # predictions compared against (n) golds and take maximum
+            em_sum += max(squad_metrics.compute_exact(a, predictions) for a in gold_answers)
+            f1_sum += max(squad_metrics.compute_f1(a, predictions) for a in gold_answers)
+    else:
+        em_sum += max(squad_metrics.compute_exact(a, predictions) for a in gold_list)
+        f1_sum += max(squad_metrics.compute_f1(a, predictions) for a in gold_list)
+
+    return {
+        "em": em_sum / max(1, len(gold_list)),
+        "f1": f1_sum / max(1, len(gold_list)),
+    }
+
+
+def process_results_qa(doc: dict[str, str], results):
+    gold_list = doc_to_target_qa(doc)
+    pred = results[0].strip().split("\n")[0]
+    scores = compute_metrics(gold_list, pred)
+    return scores

scripts/data/dialogue_data.json.gz

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:05caf10c3a95b8102a5c8eda093586daa15d7c633658520dfb1ea938172371cc
+size 1861371

scripts/data/general_data.json.gz

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b380d8c981662d5597eaa9bb5a4116971b915a35f1cbae29af9658fc8776f677
+size 1051292

scripts/data/qa_data.json.gz

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b4b67b18c37f19e12b35b4856d983a8d4d9653aaf5e9940862fd27329b92c00a
+size 1995662

scripts/data/summarization_data.json.gz

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b44a4083b14dd647c0ff7f04de0391fd3860befd0e5ca84c8492b08732270eac
+size 16445285

scripts/halueval-upload-cli.py

@@ -0,0 +1,69 @@
+#!/usr/bin/env python3
+
+import random
+import requests
+
+from datasets import load_dataset, Dataset, DatasetDict
+
+
+path = 'pminervini/HaluEval'
+
+API_URL = f"https://datasets-server.huggingface.co/splits?dataset={path}"
+response = requests.get(API_URL)
+res_json = response.json()
+
+gold_splits = {'dialogue', 'qa', 'summarization', 'general'}
+
+available_splits = {split['config'] for split in res_json['splits']} if 'splits' in res_json else set()
+
+name_to_ds = dict()
+
+for name in gold_splits:
+    ds = load_dataset("json", data_files={'data': f"data/{name}_data.json"})
+    name_to_ds[name] = ds
+    # if name not in available_splits:
+    ds.push_to_hub(path, config_name=name)
+
+def list_to_dict(lst: list) -> dict:
+    res = dict()
+    for entry in lst:
+        for k, v in entry.items():
+            if k not in res:
+                res[k] = []
+            res[k] += [v]
+    return res
+
+for name in (gold_splits - {'general'}):
+    random.seed(42)
+    ds = name_to_ds[name]
+    new_entry_lst = []
+ 
+    for entry in ds['data']:
+        is_hallucinated = random.random() > 0.5
+        if name in {'qa'}:
+            new_entry = {
+                'knowledge': entry['knowledge'],
+                'question': entry['question'],
+                'answer': entry[f'{"hallucinated" if is_hallucinated else "right"}_answer'],
+                'hallucination': 'yes' if is_hallucinated else 'no'
+            }
+            new_entry_lst += [new_entry]
+        if name in {'dialogue'}:
+            new_entry = {
+                'knowledge': entry['knowledge'],
+                'dialogue_history': entry['dialogue_history'],
+                'response': entry[f'{"hallucinated" if is_hallucinated else "right"}_response'],
+                'hallucination': 'yes' if is_hallucinated else 'no'
+            }
+        if name in {'summarization'}:
+            new_entry = {
+                'document': entry['document'],
+                'summary': entry[f'{"hallucinated" if is_hallucinated else "right"}_summary'],
+                'hallucination': 'yes' if is_hallucinated else 'no'
+            }
+
+    new_ds_map = list_to_dict(new_entry_lst)
+    new_ds = Dataset.from_dict(new_ds_map)
+    new_dsd = DatasetDict({'data': new_ds})
+
+    new_dsd.push_to_hub(path, config_name=f'{name}_samples')