Fix mismatch between instruction and label in eval.

test_of_time_accuracy.py

@@ -155,6 +155,14 @@ class TestOfTimeAccuracy(evaluate.Metric):
         except (ValueError, TypeError):
             return None
 
+    @staticmethod
+    def _fix_age_instruction_label_conflict(pred):
+        if pred is not None:
+            if "age" in pred.keys():
+                # Instruction and label conflict each other in the dataset. Fixes it.
+                pred = {"answer": pred["age"]}
+        return pred
+
     def _compute(
         self,
         predictions,
@@ -178,6 +186,7 @@ class TestOfTimeAccuracy(evaluate.Metric):
         accuracy = []
         for pred, ref in zip(predictions, references):
             if subset == "arithmetic":
+                pred = self._fix_age_instruction_label_conflict(pred)
                 pred = self._cast_prediction(ref, pred)
                 if "unordered_list" in ref:
                     pred = self._sort_unordered_list(pred)

tests/test_arithmetic_scoring.py

@@ -9,6 +9,7 @@ arithmetic_test_cases = {
         '```json\n{\n  "explanation": "The dates provided are March 2012, September 2011, June 2017, September 2019, and June 2015. These correspond to visits to Miami, Sydney, Tokyo, London, and Nairobi respectively. The latest date among these is September 2019, which is associated with London. Therefore, London is the last city visited.",\n  "malformed_unordered_list": ["Berlin","London"]\n}\n```',
         ' "To find the date of the second most important game, we need to subtract 7 days from the date of the most important game. We can do this by counting back 7 days from April 14, 2005. April 14 - 7 days = April 7, 2005", "answer": "2005-04-07"}',
         '\n```json\n{\n  "explanation": "Step 1: Determine the time it takes the robot to carry a single box. The robot takes 4 hours, 34 minutes, and 30 seconds to carry 2 boxes. We divide this time by 2 to find the time per box.\\n- Hours: 4 / 2 = 2 hours\\n- Minutes: 34 / 2 = 17 minutes\\n- Seconds: 30 / 2 = 15 seconds\\nSo, it takes the robot 2 hours, 17 minutes, and 15 seconds to carry one box.\\n\\nStep 2: Calculate the total time to carry 25 boxes. We multiply the time per box by the total number of boxes (25).\\n- Total Hours: 2 hours/box * 25 boxes = 50 hours\\n- Total Minutes: 17 minutes/box * 25 boxes = 425 minutes\\n- Total Seconds: 15 seconds/box * 25 boxes = 375 seconds\\n\\nStep 3: Convert the calculated time into the standard H:M:S format by carrying over excess seconds and minutes.\\n- Convert seconds to minutes: 375 seconds is equal to 6 minutes and 15 seconds (since 375 / 60 = 6 with a remainder of 15). We add the 6 minutes to our minutes total.\\n- New total: 50 hours, (425 + 6) minutes, 15 seconds -> 50 hours, 431 minutes, 15 seconds.\\n- Convert minutes to hours: 431 minutes is equal to 7 hours and 11 minutes (since 431 / 60 = 7 with a remainder of 11). We add the 7 hours to our hours total.\\n- New total: (50 + 7) hours, 11 minutes, 15 seconds -> 57 hours, 11 minutes, 15 seconds.\\n\\nThe final time is 57 hours, 11 minutes, and 15 seconds.",\n  "H": 57,\n  "M": 11,\n  "S": 15\n}\n```',
+        '{"explanation": "some explanation", "age": "3319"}',
     ],
     "references": [
         '{"answer": "352 BC"}',
@@ -16,9 +17,10 @@ arithmetic_test_cases = {
         '{"unordered_list": ["London", "Berlin"]}',
         '{"answer": "2005-04-07"}',
         '{"H": 57.0, "M": 11.0, "S": 15.0}',
+        '{"answer": 3319}',
     ],
-    "result": {"accuracy": 3 / 5},
-    "per_item_accuracy": [True, True, False, False, True],
+    "result": {"accuracy": 4 / 6},
+    "per_item_accuracy": [True, True, False, False, True, True],
 }