Fix SuperGlue's ReCoRD task following regression in v0.4 refactoring (E…

…leutherAI#1647)

lm_eval/tasks/super_glue/record/default.yaml

@@ -7,8 +7,9 @@ output_type: multiple_choice
 training_split: train
 validation_split: validation
 doc_to_text: !function util.doc_to_text
-doc_to_target: "{{answers}}"
-doc_to_choice: "{{entities}}"
+doc_to_target: !function util.doc_to_target
+doc_to_choice: !function util.doc_to_choice
+process_docs: !function util.process_docs
 process_results: !function util.process_results
 metric_list:
   - metric: f1
@@ -17,4 +18,4 @@ metric_list:
     higher_is_better: True
     aggregation: mean
 metadata:
-  version: 1.0
+  version: 2.0

lm_eval/tasks/super_glue/record/util.py

@@ -1,3 +1,4 @@
+import datasets
 import numpy as np
 import transformers.data.metrics.squad_metrics as squad_metrics
 
@@ -21,6 +22,21 @@ def doc_to_target(doc):
     return format_answer(query=doc["query"], entity=doc["answers"][0])
 
 
+def doc_to_choice(doc):
+    return [format_answer(query=doc["query"], entity=ans) for ans in doc["entities"]]
+
+
+def process_docs(dataset: datasets.Dataset):
+    def _process_doc(doc):
+        return {
+            "passage": doc["passage"],
+            "query": doc["query"],
+            "entities": sorted(list(set(doc["entities"]))),
+            "answers": sorted(list(set(doc["answers"]))),
+        }
+    return dataset.map(_process_doc)
+
+
 def process_results(doc, results):
     # ReCoRD's evaluation is actually deceptively simple:
     # - Pick the maximum likelihood prediction entity