Catch wrong key for unordered_list or missing dict.

test_of_time_accuracy.py

@@ -120,6 +120,11 @@ class TestOfTimeAccuracy(evaluate.Metric):
         except (ValueError, SyntaxError):
             return None
 
+    def _sort_unordered_list(self, d):
+        if isinstance(d, dict) and "unordered_list" in d:
+            return sorted(d["unordered_list"])
+        return d
+
     def _compute(
         self,
         predictions,
@@ -139,8 +144,8 @@ class TestOfTimeAccuracy(evaluate.Metric):
         accuracy = []
         for i, j in zip(predictions, references):
             if subset == "arithmetic" and "unordered_list" in j:
-                i = sorted(i["unordered_list"])
-                j = sorted(j["unordered_list"])
+                i = self._sort_unordered_list(i)
+                j = self._sort_unordered_list(j)
             accuracy.append(
                 str(i) == str(j)
             )  # Semantic subset answer JSON somestimes has int as value. Label is string.

tests.py

@@ -6,15 +6,17 @@ arithmetic_test_cases = {
     "predictions": [
         'JSON = {"explanation": "The war began in 360 BC. Since BC years count backwards, adding 8 years to 360 BC means subtracting 8 from 360, resulting in 352 BC.", "answer": "352 BC"}',
         '```json\n{\n  "explanation": "The dates provided are March 2012, September 2011, June 2017, September 2019, and June 2015. These correspond to visits to Miami, Sydney, Tokyo, London, and Nairobi respectively. The latest date among these is September 2019, which is associated with London. Therefore, London is the last city visited.",\n  "unordered_list": ["Berlin","London"]\n}\n```',
+        '```json\n{\n  "explanation": "The dates provided are March 2012, September 2011, June 2017, September 2019, and June 2015. These correspond to visits to Miami, Sydney, Tokyo, London, and Nairobi respectively. The latest date among these is September 2019, which is associated with London. Therefore, London is the last city visited.",\n  "malformed_unordered_list": ["Berlin","London"]\n}\n```',
         ' "To find the date of the second most important game, we need to subtract 7 days from the date of the most important game. We can do this by counting back 7 days from April 14, 2005. April 14 - 7 days = April 7, 2005", "answer": "2005-04-07"}',
     ],
     "references": [
         '{"answer": "352 BC"}',
         '{"unordered_list": ["London", "Berlin"]}',
+        '{"unordered_list": ["London", "Berlin"]}',
         "{'answer': '2005-04-07'}",
     ],
-    "result": {"accuracy": 2 / 3},
-    "per_item_accuracy": [True, True, False],
+    "result": {"accuracy": 0.5},
+    "per_item_accuracy": [True, True, False, False],
 }
 
 semantic_test_cases = {