From 2b26690fe5d06551030031116d073656c8dc0f33 Mon Sep 17 00:00:00 2001 From: Ben Shoham Ofir <33639234+Ofir408@users.noreply.github.com> Date: Sun, 14 Jul 2024 23:37:20 +0300 Subject: [PATCH] Added MedConceptsQA Benchmark (#2010) * Added MedConceptsQA Benchmark * pre-commit factor * update group name * update in naming * changed name * Changed mcqa to med_concepts_qa prefix * Added med_concepts_qa to README.md * Changed config files according the new format * Updated README --------- Co-authored-by: lintangsutawika --- lm_eval/tasks/README.md | 1 + lm_eval/tasks/med_concepts_qa/README.md | 49 +++++++++++++++++++ .../med_concepts_qa/_default_template_yaml | 15 ++++++ .../med_concepts_qa/_generate_configs.py | 34 +++++++++++++ .../med_concepts_qa/_med_concepts_qa.yaml | 10 ++++ .../med_concepts_qa/_med_concepts_qa_atc.yaml | 6 +++ .../_med_concepts_qa_icd10cm.yaml | 6 +++ .../_med_concepts_qa_icd10proc.yaml | 6 +++ .../_med_concepts_qa_icd9cm.yaml | 6 +++ .../_med_concepts_qa_icd9proc.yaml | 6 +++ .../med_concepts_qa_atc_easy.yaml | 5 ++ .../med_concepts_qa_atc_hard.yaml | 5 ++ .../med_concepts_qa_atc_medium.yaml | 5 ++ .../med_concepts_qa_icd10cm_easy.yaml | 5 ++ .../med_concepts_qa_icd10cm_hard.yaml | 5 ++ .../med_concepts_qa_icd10cm_medium.yaml | 5 ++ .../med_concepts_qa_icd10proc_easy.yaml | 5 ++ .../med_concepts_qa_icd10proc_hard.yaml | 5 ++ .../med_concepts_qa_icd10proc_medium.yaml | 5 ++ .../med_concepts_qa_icd9cm_easy.yaml | 5 ++ .../med_concepts_qa_icd9cm_hard.yaml | 5 ++ .../med_concepts_qa_icd9cm_medium.yaml | 5 ++ .../med_concepts_qa_icd9proc_easy.yaml | 5 ++ .../med_concepts_qa_icd9proc_hard.yaml | 5 ++ .../med_concepts_qa_icd9proc_medium.yaml | 5 ++ 25 files changed, 214 insertions(+) create mode 100644 lm_eval/tasks/med_concepts_qa/README.md create mode 100644 lm_eval/tasks/med_concepts_qa/_default_template_yaml create mode 100644 lm_eval/tasks/med_concepts_qa/_generate_configs.py create mode 100644 lm_eval/tasks/med_concepts_qa/_med_concepts_qa.yaml create mode 100644 lm_eval/tasks/med_concepts_qa/_med_concepts_qa_atc.yaml create mode 100644 lm_eval/tasks/med_concepts_qa/_med_concepts_qa_icd10cm.yaml create mode 100644 lm_eval/tasks/med_concepts_qa/_med_concepts_qa_icd10proc.yaml create mode 100644 lm_eval/tasks/med_concepts_qa/_med_concepts_qa_icd9cm.yaml create mode 100644 lm_eval/tasks/med_concepts_qa/_med_concepts_qa_icd9proc.yaml create mode 100644 lm_eval/tasks/med_concepts_qa/med_concepts_qa_atc_easy.yaml create mode 100644 lm_eval/tasks/med_concepts_qa/med_concepts_qa_atc_hard.yaml create mode 100644 lm_eval/tasks/med_concepts_qa/med_concepts_qa_atc_medium.yaml create mode 100644 lm_eval/tasks/med_concepts_qa/med_concepts_qa_icd10cm_easy.yaml create mode 100644 lm_eval/tasks/med_concepts_qa/med_concepts_qa_icd10cm_hard.yaml create mode 100644 lm_eval/tasks/med_concepts_qa/med_concepts_qa_icd10cm_medium.yaml create mode 100644 lm_eval/tasks/med_concepts_qa/med_concepts_qa_icd10proc_easy.yaml create mode 100644 lm_eval/tasks/med_concepts_qa/med_concepts_qa_icd10proc_hard.yaml create mode 100644 lm_eval/tasks/med_concepts_qa/med_concepts_qa_icd10proc_medium.yaml create mode 100644 lm_eval/tasks/med_concepts_qa/med_concepts_qa_icd9cm_easy.yaml create mode 100644 lm_eval/tasks/med_concepts_qa/med_concepts_qa_icd9cm_hard.yaml create mode 100644 lm_eval/tasks/med_concepts_qa/med_concepts_qa_icd9cm_medium.yaml create mode 100644 lm_eval/tasks/med_concepts_qa/med_concepts_qa_icd9proc_easy.yaml create mode 100644 lm_eval/tasks/med_concepts_qa/med_concepts_qa_icd9proc_hard.yaml create mode 100644 lm_eval/tasks/med_concepts_qa/med_concepts_qa_icd9proc_medium.yaml diff --git a/lm_eval/tasks/README.md b/lm_eval/tasks/README.md index c6e4c4d73f..b088bddd51 100644 --- a/lm_eval/tasks/README.md +++ b/lm_eval/tasks/README.md @@ -62,6 +62,7 @@ | [logiqa2](logiqa2/README.md) | Large-scale logical reasoning dataset adapted from the Chinese Civil Service Examination. | English, Chinese | | [mathqa](mathqa/README.md) | Question answering tasks involving mathematical reasoning and problem-solving. | English | | [mc_taco](mc_taco/README.md) | Question-answer pairs that require temporal commonsense comprehension. | English | +| [med_concepts_qa](med_concepts_qa/README.md) | Benchmark for evaluating LLMs on their abilities to interpret medical codes and distinguish between medical concept. | English | | medmcqa | Medical multiple choice questions assessing detailed medical knowledge. | English | | medqa | Multiple choice question answering based on the United States Medical License Exams. | | | [mgsm](mgsm/README.md) | Benchmark of multilingual grade-school math problems. | Spanish, French, German, Russian, Chinese, Japanese, Thai, Swahili, Bengali, Telugu | diff --git a/lm_eval/tasks/med_concepts_qa/README.md b/lm_eval/tasks/med_concepts_qa/README.md new file mode 100644 index 0000000000..666d6446b1 --- /dev/null +++ b/lm_eval/tasks/med_concepts_qa/README.md @@ -0,0 +1,49 @@ +# MedConceptsQA + +### Paper + +Title: `MedConceptsQA: Open Source Medical Concepts QA Benchmark` + +Abstract: https://arxiv.org/abs/2405.07348 + +MedConceptsQA is a dedicated open source benchmark for medical concepts question answering. The benchmark comprises of questions of various medical concepts across different vocabularies: diagnoses, procedures, and drugs. + +The questions are categorized into three levels of difficulty: easy, medium, and hard. + +Our benchmark serves as a valuable resource for evaluating the +abilities of Large Language Models to interpret medical codes and distinguish +between medical concepts. + +### Citation + +``` +@article{shoham2024medconceptsqa, + title={MedConceptsQA--Open Source Medical Concepts QA Benchmark}, + author={Shoham, Ofir Ben and Rappoport, Nadav}, + journal={arXiv preprint arXiv:2405.07348}, + year={2024} +} +``` + +### Groups and Tasks + +#### Groups + +* `med_concepts_qa`: Contains all the QA tasks (diagnosis, procedures ,and drugs). + +#### Tasks + + +* `med_concepts_qa_icd9cm` - ICD9-CM (diagnosis codes, ICD9 format) question-answering. This involves providing information, clarifications, and answering questions related to ICD-9-CM (International Classification of Diseases, 9th Revision, Clinical Modification) diagnosis codes. + + +* `med_concepts_qa_icd10cm` - ICD10-CM (diagnosis codes, ICD10 format) question-answering. This involves providing information, clarifications, and answering questions related to ICD-10-CM (International Classification of Diseases, 10th Revision, Clinical Modification) diagnosis codes. + + +* `med_concepts_qa_icd9proc` - ICD9-Proc (procedure codes, ICD9 format) question-answering. This involves providing information, clarifications, and answering questions related to ICD-9-PCS (International Classification of Diseases, 9th Revision, Procedure Coding System) procedure codes. + + +* `med_concepts_qa_icd10proc` - ICD10-Proc (procedure codes, ICD10 format) question-answering. This involves providing information, clarifications, and answering questions related to ICD-10-PCS (International Classification of Diseases, 10th Revision, Procedure Coding System) procedure codes. + + +* `med_concepts_qa_atc` - ATC (Anatomical Therapeutic Chemical Classification System) question-answering. This involves providing information, clarifications, and answering questions related to the ATC classification system, which is used for the classification of drugs and other medical products according to the organ or system on which they act and their therapeutic, pharmacological, and chemical properties. diff --git a/lm_eval/tasks/med_concepts_qa/_default_template_yaml b/lm_eval/tasks/med_concepts_qa/_default_template_yaml new file mode 100644 index 0000000000..84198fa7b6 --- /dev/null +++ b/lm_eval/tasks/med_concepts_qa/_default_template_yaml @@ -0,0 +1,15 @@ +dataset_path: ofir408/MedConceptsQA +output_type: multiple_choice +description: "Answer A,B,C,D according to the answer to this multiple choice question.\n" +fewshot_split: dev +fewshot_config: + sampler: first_n +num_fewshot: 4 +test_split: test +doc_to_text: "{{question}}\nAnswer:" +doc_to_target: answer_id +doc_to_choice: ['A', 'B', 'C', 'D'] +metric_list: + - metric: acc + aggregation: mean + higher_is_better: true diff --git a/lm_eval/tasks/med_concepts_qa/_generate_configs.py b/lm_eval/tasks/med_concepts_qa/_generate_configs.py new file mode 100644 index 0000000000..fd81edbbd0 --- /dev/null +++ b/lm_eval/tasks/med_concepts_qa/_generate_configs.py @@ -0,0 +1,34 @@ +from typing import List + +import yaml + + +def generate_yaml_content(vocab_name: str, level: str): + content = { + "dataset_name": f"{vocab_name}_{level}", + "tag": f"med_concepts_qa_{vocab_name}_tasks", + "include": "_default_template_yaml", + "task": f"med_concepts_qa_{vocab_name}_{level}", + "task_alias": f"{vocab_name}_{level}", + } + return content + + +def generate_yaml_files( + vocab_names: List[str], levels: List[str], file_name_prefix: str +): + for vocab_name in vocab_names: + for level in levels: + yaml_content = generate_yaml_content(vocab_name, level) + filename = f"{file_name_prefix}_{vocab_name}_{level}.yaml" + with open(filename, "w") as yaml_file: + yaml.dump(yaml_content, yaml_file, default_flow_style=False) + print(f"Done to generated {filename}") + + +if __name__ == "__main__": + generate_yaml_files( + vocab_names=["icd9cm", "icd10cm", "icd9proc", "icd10proc", "atc"], + levels=["easy", "medium", "hard"], + file_name_prefix="med_concepts_qa", + ) diff --git a/lm_eval/tasks/med_concepts_qa/_med_concepts_qa.yaml b/lm_eval/tasks/med_concepts_qa/_med_concepts_qa.yaml new file mode 100644 index 0000000000..c737127f98 --- /dev/null +++ b/lm_eval/tasks/med_concepts_qa/_med_concepts_qa.yaml @@ -0,0 +1,10 @@ +group: med_concepts_qa +task: + - med_concepts_qa_icd9cm + - med_concepts_qa_icd10cm + - med_concepts_qa_icd9proc + - med_concepts_qa_icd10proc + - med_concepts_qa_atc +aggregate_metric_list: + - metric: acc + aggregation: mean diff --git a/lm_eval/tasks/med_concepts_qa/_med_concepts_qa_atc.yaml b/lm_eval/tasks/med_concepts_qa/_med_concepts_qa_atc.yaml new file mode 100644 index 0000000000..b457ae582f --- /dev/null +++ b/lm_eval/tasks/med_concepts_qa/_med_concepts_qa_atc.yaml @@ -0,0 +1,6 @@ +group: med_concepts_qa_atc +task: + - med_concepts_qa_atc_tasks +aggregate_metric_list: + - metric: acc + aggregation: mean \ No newline at end of file diff --git a/lm_eval/tasks/med_concepts_qa/_med_concepts_qa_icd10cm.yaml b/lm_eval/tasks/med_concepts_qa/_med_concepts_qa_icd10cm.yaml new file mode 100644 index 0000000000..c4afc1bdfe --- /dev/null +++ b/lm_eval/tasks/med_concepts_qa/_med_concepts_qa_icd10cm.yaml @@ -0,0 +1,6 @@ +group: med_concepts_qa_icd10cm +task: + - med_concepts_qa_icd10cm_tasks +aggregate_metric_list: + - metric: acc + aggregation: mean diff --git a/lm_eval/tasks/med_concepts_qa/_med_concepts_qa_icd10proc.yaml b/lm_eval/tasks/med_concepts_qa/_med_concepts_qa_icd10proc.yaml new file mode 100644 index 0000000000..777a5ce6ee --- /dev/null +++ b/lm_eval/tasks/med_concepts_qa/_med_concepts_qa_icd10proc.yaml @@ -0,0 +1,6 @@ +group: med_concepts_qa_icd10proc +task: + - med_concepts_qa_icd10proc_tasks +aggregate_metric_list: + - metric: acc + aggregation: mean \ No newline at end of file diff --git a/lm_eval/tasks/med_concepts_qa/_med_concepts_qa_icd9cm.yaml b/lm_eval/tasks/med_concepts_qa/_med_concepts_qa_icd9cm.yaml new file mode 100644 index 0000000000..d5671bf035 --- /dev/null +++ b/lm_eval/tasks/med_concepts_qa/_med_concepts_qa_icd9cm.yaml @@ -0,0 +1,6 @@ +group: med_concepts_qa_icd9cm +task: + - med_concepts_qa_icd9cm_tasks +aggregate_metric_list: + - metric: acc + aggregation: mean \ No newline at end of file diff --git a/lm_eval/tasks/med_concepts_qa/_med_concepts_qa_icd9proc.yaml b/lm_eval/tasks/med_concepts_qa/_med_concepts_qa_icd9proc.yaml new file mode 100644 index 0000000000..4487772fa1 --- /dev/null +++ b/lm_eval/tasks/med_concepts_qa/_med_concepts_qa_icd9proc.yaml @@ -0,0 +1,6 @@ +group: med_concepts_qa_icd9proc +task: + - med_concepts_qa_icd9proc_tasks +aggregate_metric_list: + - metric: acc + aggregation: mean \ No newline at end of file diff --git a/lm_eval/tasks/med_concepts_qa/med_concepts_qa_atc_easy.yaml b/lm_eval/tasks/med_concepts_qa/med_concepts_qa_atc_easy.yaml new file mode 100644 index 0000000000..be2f62a1dc --- /dev/null +++ b/lm_eval/tasks/med_concepts_qa/med_concepts_qa_atc_easy.yaml @@ -0,0 +1,5 @@ +dataset_name: atc_easy +include: _default_template_yaml +tag: med_concepts_qa_atc_tasks +task: med_concepts_qa_atc_easy +task_alias: atc_easy diff --git a/lm_eval/tasks/med_concepts_qa/med_concepts_qa_atc_hard.yaml b/lm_eval/tasks/med_concepts_qa/med_concepts_qa_atc_hard.yaml new file mode 100644 index 0000000000..a54d27d976 --- /dev/null +++ b/lm_eval/tasks/med_concepts_qa/med_concepts_qa_atc_hard.yaml @@ -0,0 +1,5 @@ +dataset_name: atc_hard +include: _default_template_yaml +tag: med_concepts_qa_atc_tasks +task: med_concepts_qa_atc_hard +task_alias: atc_hard diff --git a/lm_eval/tasks/med_concepts_qa/med_concepts_qa_atc_medium.yaml b/lm_eval/tasks/med_concepts_qa/med_concepts_qa_atc_medium.yaml new file mode 100644 index 0000000000..e0b64ac85b --- /dev/null +++ b/lm_eval/tasks/med_concepts_qa/med_concepts_qa_atc_medium.yaml @@ -0,0 +1,5 @@ +dataset_name: atc_medium +include: _default_template_yaml +tag: med_concepts_qa_atc_tasks +task: med_concepts_qa_atc_medium +task_alias: atc_medium diff --git a/lm_eval/tasks/med_concepts_qa/med_concepts_qa_icd10cm_easy.yaml b/lm_eval/tasks/med_concepts_qa/med_concepts_qa_icd10cm_easy.yaml new file mode 100644 index 0000000000..0fe8da7ddd --- /dev/null +++ b/lm_eval/tasks/med_concepts_qa/med_concepts_qa_icd10cm_easy.yaml @@ -0,0 +1,5 @@ +dataset_name: icd10cm_easy +include: _default_template_yaml +tag: med_concepts_qa_icd10cm_tasks +task: med_concepts_qa_icd10cm_easy +task_alias: icd10cm_easy diff --git a/lm_eval/tasks/med_concepts_qa/med_concepts_qa_icd10cm_hard.yaml b/lm_eval/tasks/med_concepts_qa/med_concepts_qa_icd10cm_hard.yaml new file mode 100644 index 0000000000..2b291b0eb4 --- /dev/null +++ b/lm_eval/tasks/med_concepts_qa/med_concepts_qa_icd10cm_hard.yaml @@ -0,0 +1,5 @@ +dataset_name: icd10cm_hard +include: _default_template_yaml +tag: med_concepts_qa_icd10cm_tasks +task: med_concepts_qa_icd10cm_hard +task_alias: icd10cm_hard diff --git a/lm_eval/tasks/med_concepts_qa/med_concepts_qa_icd10cm_medium.yaml b/lm_eval/tasks/med_concepts_qa/med_concepts_qa_icd10cm_medium.yaml new file mode 100644 index 0000000000..b1678fa311 --- /dev/null +++ b/lm_eval/tasks/med_concepts_qa/med_concepts_qa_icd10cm_medium.yaml @@ -0,0 +1,5 @@ +dataset_name: icd10cm_medium +include: _default_template_yaml +tag: med_concepts_qa_icd10cm_tasks +task: med_concepts_qa_icd10cm_medium +task_alias: icd10cm_medium diff --git a/lm_eval/tasks/med_concepts_qa/med_concepts_qa_icd10proc_easy.yaml b/lm_eval/tasks/med_concepts_qa/med_concepts_qa_icd10proc_easy.yaml new file mode 100644 index 0000000000..301f559d7c --- /dev/null +++ b/lm_eval/tasks/med_concepts_qa/med_concepts_qa_icd10proc_easy.yaml @@ -0,0 +1,5 @@ +dataset_name: icd10proc_easy +include: _default_template_yaml +tag: med_concepts_qa_icd10proc_tasks +task: med_concepts_qa_icd10proc_easy +task_alias: icd10proc_easy diff --git a/lm_eval/tasks/med_concepts_qa/med_concepts_qa_icd10proc_hard.yaml b/lm_eval/tasks/med_concepts_qa/med_concepts_qa_icd10proc_hard.yaml new file mode 100644 index 0000000000..15efafba09 --- /dev/null +++ b/lm_eval/tasks/med_concepts_qa/med_concepts_qa_icd10proc_hard.yaml @@ -0,0 +1,5 @@ +dataset_name: icd10proc_hard +include: _default_template_yaml +tag: med_concepts_qa_icd10proc_tasks +task: med_concepts_qa_icd10proc_hard +task_alias: icd10proc_hard diff --git a/lm_eval/tasks/med_concepts_qa/med_concepts_qa_icd10proc_medium.yaml b/lm_eval/tasks/med_concepts_qa/med_concepts_qa_icd10proc_medium.yaml new file mode 100644 index 0000000000..648bd46075 --- /dev/null +++ b/lm_eval/tasks/med_concepts_qa/med_concepts_qa_icd10proc_medium.yaml @@ -0,0 +1,5 @@ +dataset_name: icd10proc_medium +include: _default_template_yaml +tag: med_concepts_qa_icd10proc_tasks +task: med_concepts_qa_icd10proc_medium +task_alias: icd10proc_medium diff --git a/lm_eval/tasks/med_concepts_qa/med_concepts_qa_icd9cm_easy.yaml b/lm_eval/tasks/med_concepts_qa/med_concepts_qa_icd9cm_easy.yaml new file mode 100644 index 0000000000..514a9e2586 --- /dev/null +++ b/lm_eval/tasks/med_concepts_qa/med_concepts_qa_icd9cm_easy.yaml @@ -0,0 +1,5 @@ +dataset_name: icd9cm_easy +include: _default_template_yaml +tag: med_concepts_qa_icd9cm_tasks +task: med_concepts_qa_icd9cm_easy +task_alias: icd9cm_easy diff --git a/lm_eval/tasks/med_concepts_qa/med_concepts_qa_icd9cm_hard.yaml b/lm_eval/tasks/med_concepts_qa/med_concepts_qa_icd9cm_hard.yaml new file mode 100644 index 0000000000..41d0fd5534 --- /dev/null +++ b/lm_eval/tasks/med_concepts_qa/med_concepts_qa_icd9cm_hard.yaml @@ -0,0 +1,5 @@ +dataset_name: icd9cm_hard +include: _default_template_yaml +tag: med_concepts_qa_icd9cm_tasks +task: med_concepts_qa_icd9cm_hard +task_alias: icd9cm_hard diff --git a/lm_eval/tasks/med_concepts_qa/med_concepts_qa_icd9cm_medium.yaml b/lm_eval/tasks/med_concepts_qa/med_concepts_qa_icd9cm_medium.yaml new file mode 100644 index 0000000000..90f93f9bc2 --- /dev/null +++ b/lm_eval/tasks/med_concepts_qa/med_concepts_qa_icd9cm_medium.yaml @@ -0,0 +1,5 @@ +dataset_name: icd9cm_medium +include: _default_template_yaml +tag: med_concepts_qa_icd9cm_tasks +task: med_concepts_qa_icd9cm_medium +task_alias: icd9cm_medium diff --git a/lm_eval/tasks/med_concepts_qa/med_concepts_qa_icd9proc_easy.yaml b/lm_eval/tasks/med_concepts_qa/med_concepts_qa_icd9proc_easy.yaml new file mode 100644 index 0000000000..459fa464ff --- /dev/null +++ b/lm_eval/tasks/med_concepts_qa/med_concepts_qa_icd9proc_easy.yaml @@ -0,0 +1,5 @@ +dataset_name: icd9proc_easy +include: _default_template_yaml +tag: med_concepts_qa_icd9proc_tasks +task: med_concepts_qa_icd9proc_easy +task_alias: icd9proc_easy diff --git a/lm_eval/tasks/med_concepts_qa/med_concepts_qa_icd9proc_hard.yaml b/lm_eval/tasks/med_concepts_qa/med_concepts_qa_icd9proc_hard.yaml new file mode 100644 index 0000000000..5424623c30 --- /dev/null +++ b/lm_eval/tasks/med_concepts_qa/med_concepts_qa_icd9proc_hard.yaml @@ -0,0 +1,5 @@ +dataset_name: icd9proc_hard +include: _default_template_yaml +tag: med_concepts_qa_icd9proc_tasks +task: med_concepts_qa_icd9proc_hard +task_alias: icd9proc_hard diff --git a/lm_eval/tasks/med_concepts_qa/med_concepts_qa_icd9proc_medium.yaml b/lm_eval/tasks/med_concepts_qa/med_concepts_qa_icd9proc_medium.yaml new file mode 100644 index 0000000000..8430292099 --- /dev/null +++ b/lm_eval/tasks/med_concepts_qa/med_concepts_qa_icd9proc_medium.yaml @@ -0,0 +1,5 @@ +dataset_name: icd9proc_medium +include: _default_template_yaml +tag: med_concepts_qa_icd9proc_tasks +task: med_concepts_qa_icd9proc_medium +task_alias: icd9proc_medium