[SIG] add LogiQA dataset

configs/datasets/LogiQA/LogiQA_en_ppl.py

@@ -0,0 +1,4 @@
+from mmengine.config import read_base
+
+with read_base():
+    from .LogiQA_en_ppl_20dfb3 import LogiQA_en_datasets  # noqa: F401, F403

configs/datasets/LogiQA/LogiQA_en_ppl_20dfb3.py

@@ -0,0 +1,48 @@
+from opencompass.openicl.icl_prompt_template import PromptTemplate
+from opencompass.openicl.icl_retriever import FixKRetriever
+from opencompass.openicl.icl_inferencer import PPLInferencer
+from opencompass.openicl.icl_evaluator import AccEvaluator
+from opencompass.datasets import LogiQADataset
+
+
+_hint = "The following are logical reasoning questions from the Chinese National " \
+    "Civil Service Examination. Please choose the correct answer.\n"
+LogiQA_en_infer_cfg = dict(
+    ice_template=dict(
+        type=PromptTemplate,
+        template="Context: {context}\nQuery: {query}\nA. {A}\nB. {B}\nC. {C}\nD. {D}\nAnswer: {correct_option}",
+    ),
+    prompt_template=dict(
+        type=PromptTemplate,
+        template={
+            answer:
+            f"{_hint}</E>Context: {{context}}\nQuery: {{query}}\nA. {{A}}\nB. {{B}}\nC. {{C}}\nD. {{D}}\nAnswer: {answer}"
+            for answer in ['A', 'B', 'C', 'D']
+        },
+        ice_token='</E>',
+    ),
+    retriever=dict(type=FixKRetriever, fix_id_list=[0, 1, 2, 3, 4]),
+    inferencer=dict(type=PPLInferencer))
+
+LogiQA_en_eval_cfg = dict(evaluator=dict(type=AccEvaluator), )
+
+
+LogiQA_en_datasets = []
+for _split in ["validation", "test"]:
+
+    LogiQA_en_reader_cfg = dict(
+        input_columns=['context', 'query', 'A', 'B', 'C', 'D'],
+        output_column='correct_option',
+        test_split=_split
+    )
+
+    LogiQA_en_datasets.append(
+        dict(
+            abbr=f'LogiQA_en-{_split}',
+            type=LogiQADataset,
+            path='lucasmccabe/logiqa',
+            reader_cfg=LogiQA_en_reader_cfg,
+            infer_cfg=LogiQA_en_infer_cfg,
+            eval_cfg=LogiQA_en_eval_cfg
+        )
+    )

configs/datasets/LogiQA/LogiQA_zh_ppl.py

@@ -0,0 +1,4 @@
+from mmengine.config import read_base
+
+with read_base():
+    from .LogiQA_zh_ppl_19fd62 import LogiQA_zh_datasets  # noqa: F401, F403

configs/datasets/LogiQA/LogiQA_zh_ppl_19fd62.py

@@ -0,0 +1,47 @@
+from opencompass.openicl.icl_prompt_template import PromptTemplate
+from opencompass.openicl.icl_retriever import FixKRetriever
+from opencompass.openicl.icl_inferencer import PPLInferencer
+from opencompass.openicl.icl_evaluator import AccEvaluator
+from opencompass.datasets import LogiQADataset
+
+
+_hint = "以下是中国国家公务员考试的逻辑推理题，请选出其中的正确答案。\n"
+LogiQA_zh_infer_cfg = dict(
+    ice_template=dict(
+        type=PromptTemplate,
+        template="上下文信息: {context}\n提问: {query}\nA. {A}\nB. {B}\nC. {C}\nD. {D}\n答案: {correct_option}",
+    ),
+    prompt_template=dict(
+        type=PromptTemplate,
+        template={
+            answer:
+            f"{_hint}</E>上下文信息: {{context}}\n提问: {{query}}\nA. {{A}}\nB. {{B}}\nC. {{C}}\nD. {{D}}\n答案: {answer}"
+            for answer in ['A', 'B', 'C', 'D']
+        },
+        ice_token='</E>',
+    ),
+    retriever=dict(type=FixKRetriever, fix_id_list=[0, 1, 2, 3, 4]),
+    inferencer=dict(type=PPLInferencer))
+
+LogiQA_zh_eval_cfg = dict(evaluator=dict(type=AccEvaluator), )
+
+
+LogiQA_zh_datasets = []
+for _split in ["validation", "test"]:
+
+    LogiQA_zh_reader_cfg = dict(
+        input_columns=['context', 'query', 'A', 'B', 'C', 'D'],
+        output_column='correct_option',
+        test_split=_split
+    )
+
+    LogiQA_zh_datasets.append(
+        dict(
+            abbr=f'LogiQA_zh-{_split}',
+            type=LogiQADataset,
+            path='jiacheng-ye/logiqa-zh',
+            reader_cfg=LogiQA_zh_reader_cfg,
+            infer_cfg=LogiQA_zh_infer_cfg,
+            eval_cfg=LogiQA_zh_eval_cfg
+        )
+    )

opencompass/datasets/__init__.py

@@ -43,6 +43,7 @@
 from .lawbench import *  # noqa: F401, F403
 from .lcsts import *  # noqa: F401, F403
 from .leval import *  # noqa: F401, F403
+from .logiqa import *  # noqa: F401, F403
 from .longbench import *  # noqa: F401, F403
 from .math import *  # noqa: F401, F403
 from .mathbench import *  # noqa: F401, F403

opencompass/datasets/logiqa.py

@@ -0,0 +1,25 @@
+from datasets import load_dataset
+
+from opencompass.registry import LOAD_DATASET
+
+from .base import BaseDataset
+
+
+@LOAD_DATASET.register_module()
+class LogiQADataset(BaseDataset):
+
+    @staticmethod
+    def load(path: str):
+        dataset = load_dataset(path=path)
+
+        def preprocess(example):
+            options = example.pop('options')
+            example['A'] = options[0]
+            example['B'] = options[1]
+            example['C'] = options[2]
+            example['D'] = options[3]
+            example['correct_option'] = chr(65 + example['correct_option'])
+            return example
+
+        dataset = dataset.map(preprocess)
+        return dataset