Correctly Print Task Versioning (EleutherAI#1173)

* change version field formatting in metadata * mention versioning in new task guide * add instructions for changelog * run linters
OpenLLM-France · Dec 21, 2023 · 9cd7989 · 9cd7989
1 parent a0cfe3f
commit 9cd7989
Show file tree

Hide file tree

Showing 128 changed files with 150 additions and 125 deletions.
diff --git a/docs/new_task_guide.md b/docs/new_task_guide.md
@@ -315,6 +315,25 @@ python -m scripts.write_out \
 Open the file specified at the `--output_base_path <path>` and ensure it passes
 a simple eye test.
 
+## Versioning
+
+One key feature in LM Evaluation Harness is the ability to version tasks--that is, mark them with a specific version number that can be bumped whenever a breaking change is made.
+
+This version info can be provided by adding the following to your new task config file:
+
+```
+metadata:
+  version: 0
+```
+
+Now, whenever a change needs to be made to your task in the future, please increase the version number by 1 so that users can differentiate the different task iterations and versions.
+
+If you are incrementing a task's version, please also consider adding a changelog to the task's README.md noting the date, PR number, what version you have updated to, and a one-liner describing the change.
+
+for example,
+
+* \[Dec 25, 2023\] (PR #999) Version 0.0 -> 1.0: Fixed a bug with answer extraction that led to underestimated performance.
+
 ## Checking performance + equivalence
 
 It's now time to check models' performance on your task! In the evaluation harness, we intend to support a wide range of evaluation tasks and setups, but prioritize the inclusion of already-proven benchmarks following the precise evaluation setups in the literature where possible.

diff --git a/lm_eval/tasks/anli/anli_r1.yaml b/lm_eval/tasks/anli/anli_r1.yaml
@@ -23,4 +23,4 @@ metric_list:
     aggregation: mean
     higher_is_better: true
 metadata:
-  - version: 1.0
+  version: 1.0
diff --git a/lm_eval/tasks/arc/arc_easy.yaml b/lm_eval/tasks/arc/arc_easy.yaml
@@ -20,4 +20,4 @@ metric_list:
     aggregation: mean
     higher_is_better: true
 metadata:
-  - version: 1.0
+  version: 1.0
diff --git a/lm_eval/tasks/arithmetic/arithmetic_1dc.yaml b/lm_eval/tasks/arithmetic/arithmetic_1dc.yaml
@@ -13,4 +13,4 @@ metric_list:
     aggregation: mean
     higher_is_better: true
 metadata:
-  - version: 1.0
+  version: 1.0
diff --git a/lm_eval/tasks/asdiv/default.yaml b/lm_eval/tasks/asdiv/default.yaml
@@ -11,4 +11,4 @@ metric_list:
     aggregation: mean
     higher_is_better: true
 metadata:
-  - version: 1.0
+  version: 1.0
diff --git a/lm_eval/tasks/babi/babi.yaml b/lm_eval/tasks/babi/babi.yaml
@@ -17,4 +17,4 @@ metric_list:
     aggregation: mean
     higher_is_better: true
 metadata:
-  - version: 0.0
+  version: 0.0
diff --git a/lm_eval/tasks/bbh/cot_fewshot/_cot_fewshot_template_yaml b/lm_eval/tasks/bbh/cot_fewshot/_cot_fewshot_template_yaml
@@ -27,4 +27,4 @@ filter_list:
       - function: "take_first"
 num_fewshot: 0
 metadata:
-  - version: 1.0
+  version: 1.0
diff --git a/lm_eval/tasks/bbh/cot_zeroshot/_cot_zeroshot_template_yaml b/lm_eval/tasks/bbh/cot_zeroshot/_cot_zeroshot_template_yaml
@@ -24,4 +24,4 @@ filter_list:
       - function: "take_first"
 num_fewshot: 0
 metadata:
-  - version: 0
+  version: 0
diff --git a/lm_eval/tasks/bbh/fewshot/_fewshot_template_yaml b/lm_eval/tasks/bbh/fewshot/_fewshot_template_yaml
@@ -18,4 +18,4 @@ generation_kwargs:
   temperature: 0.0
 num_fewshot: 0
 metadata:
-  - version: 0
+  version: 0
diff --git a/lm_eval/tasks/bbh/zeroshot/_zeroshot_template_yaml b/lm_eval/tasks/bbh/zeroshot/_zeroshot_template_yaml
@@ -18,4 +18,4 @@ generation_kwargs:
   temperature: 0.0
 num_fewshot: 0
 metadata:
-  - version: 0
+  version: 0
diff --git a/lm_eval/tasks/belebele/_default_template_yaml b/lm_eval/tasks/belebele/_default_template_yaml
@@ -18,4 +18,4 @@ metric_list:
     aggregation: mean
     higher_is_better: true
 metadata:
-  - version: 0.0
+  version: 0.0
diff --git a/lm_eval/tasks/bigbench/generate_until_template_yaml b/lm_eval/tasks/bigbench/generate_until_template_yaml
@@ -15,4 +15,4 @@ metric_list:
     higher_is_better: true
     ignore_punctuation: true
 metadata:
-  - version: 0.0
+  version: 0.0
diff --git a/lm_eval/tasks/bigbench/multiple_choice/causal_judgement.yaml b/lm_eval/tasks/bigbench/multiple_choice/causal_judgement.yaml
@@ -0,0 +1,4 @@
+# Generated by utils.py
+dataset_name: causal_judgment_zero_shot
+include: ../multiple_choice_template_yaml
+task: bigbench_causal_judgement_multiple_choice
diff --git a/lm_eval/tasks/bigbench/multiple_choice_template_yaml b/lm_eval/tasks/bigbench/multiple_choice_template_yaml
@@ -12,4 +12,4 @@ metric_list:
   - metric: acc
   # TODO: brier score and other metrics
 metadata:
-  - version: 0.0
+  version: 0.0
diff --git a/lm_eval/tasks/blimp/_template_yaml b/lm_eval/tasks/blimp/_template_yaml
@@ -11,4 +11,4 @@ doc_to_decontamination_query: "{{sentence_good}} {{sentence_bad}}"
 metric_list:
   - metric: acc
 metadata:
-  - version: 1.0
+  version: 1.0
diff --git a/lm_eval/tasks/ceval/_default_ceval_yaml b/lm_eval/tasks/ceval/_default_ceval_yaml
@@ -16,4 +16,4 @@ metric_list:
     aggregation: mean
     higher_is_better: true
 metadata:
-  - version: 1.0
+  version: 1.0
diff --git a/lm_eval/tasks/cmmlu/_default_template_yaml b/lm_eval/tasks/cmmlu/_default_template_yaml
@@ -16,4 +16,4 @@ metric_list:
     aggregation: mean
     higher_is_better: true
 metadata:
-  - version: 0.0
+  version: 0.0
diff --git a/lm_eval/tasks/code_x_glue/code-text/go.yaml b/lm_eval/tasks/code_x_glue/code-text/go.yaml
@@ -18,4 +18,4 @@ metric_list:
     aggregation: mean
     higher_is_better: True
 metadata:
-  - version: 0.0
+  version: 0.0
diff --git a/lm_eval/tasks/code_x_glue/code-text/java.yaml b/lm_eval/tasks/code_x_glue/code-text/java.yaml
@@ -18,4 +18,4 @@ metric_list:
     aggregation: mean
     higher_is_better: True
 metadata:
-  - version: 0.0
+  version: 0.0
diff --git a/lm_eval/tasks/code_x_glue/code-text/javascript.yaml b/lm_eval/tasks/code_x_glue/code-text/javascript.yaml
@@ -18,4 +18,4 @@ metric_list:
     aggregation: mean
     higher_is_better: True
 metadata:
-  - version: 0.0
+  version: 0.0
diff --git a/lm_eval/tasks/code_x_glue/code-text/php.yaml b/lm_eval/tasks/code_x_glue/code-text/php.yaml
@@ -18,4 +18,4 @@ metric_list:
     aggregation: mean
     higher_is_better: True
 metadata:
-  - version: 0.0
+  version: 0.0
diff --git a/lm_eval/tasks/code_x_glue/code-text/python.yaml b/lm_eval/tasks/code_x_glue/code-text/python.yaml
@@ -18,4 +18,4 @@ metric_list:
     aggregation: mean
     higher_is_better: True
 metadata:
-  - version: 0.0
+  version: 0.0
diff --git a/lm_eval/tasks/code_x_glue/code-text/ruby.yaml b/lm_eval/tasks/code_x_glue/code-text/ruby.yaml
@@ -18,4 +18,4 @@ metric_list:
     aggregation: mean
     higher_is_better: True
 metadata:
-  - version: 2.0
+  version: 2.0
diff --git a/lm_eval/tasks/coqa/default.yaml b/lm_eval/tasks/coqa/default.yaml
@@ -19,4 +19,4 @@ metric_list:
     aggregation: mean
     higher_is_better: true
 metadata:
-  - version: 2.0
+  version: 2.0
diff --git a/lm_eval/tasks/crows_pairs/crows_pairs_english.yaml b/lm_eval/tasks/crows_pairs/crows_pairs_english.yaml
@@ -20,4 +20,4 @@ metric_list:
     aggregation: mean
     higher_is_better: false
 metadata:
-  - version: 1.0
+  version: 1.0
diff --git a/lm_eval/tasks/csatqa/_default_csatqa_yaml b/lm_eval/tasks/csatqa/_default_csatqa_yaml
@@ -14,4 +14,4 @@ metric_list:
     aggregation: mean
     higher_is_better: true
 metadata:
-  - version: 0.0
+  version: 0.0
diff --git a/lm_eval/tasks/drop/default.yaml b/lm_eval/tasks/drop/default.yaml
@@ -21,4 +21,4 @@ metric_list:
     aggregation: mean
     higher_is_better: true
 metadata:
-  - version: 2.0
+  version: 2.0
diff --git a/lm_eval/tasks/fld/fld_default.yaml b/lm_eval/tasks/fld/fld_default.yaml
@@ -12,3 +12,5 @@ metric_list:
   - metric: exact_match
     aggregation: mean
     higher_is_better: true
+metadata:
+  version: 0.0
diff --git a/lm_eval/tasks/glue/cola/default.yaml b/lm_eval/tasks/glue/cola/default.yaml
@@ -13,4 +13,4 @@ doc_to_decontamination_query: sentence
 metric_list:
   - metric: mcc
 metadata:
-  - version: 1.0
+  version: 1.0
diff --git a/lm_eval/tasks/glue/mnli/default.yaml b/lm_eval/tasks/glue/mnli/default.yaml
@@ -11,4 +11,4 @@ doc_to_choice: ["True", "Neither", "False"]
 metric_list:
   - metric: acc
 metadata:
-  - version: 1.0
+  version: 1.0
diff --git a/lm_eval/tasks/glue/mrpc/default.yaml b/lm_eval/tasks/glue/mrpc/default.yaml
@@ -12,4 +12,4 @@ metric_list:
   - metric: acc
   - metric: f1
 metadata:
-  - version: 1.0
+  version: 1.0
diff --git a/lm_eval/tasks/glue/qnli/default.yaml b/lm_eval/tasks/glue/qnli/default.yaml
@@ -11,4 +11,4 @@ doc_to_choice: ["yes", "no"]
 metric_list:
   - metric: acc
 metadata:
-  - version: 1.0
+  version: 1.0
diff --git a/lm_eval/tasks/glue/qqp/default.yaml b/lm_eval/tasks/glue/qqp/default.yaml
@@ -12,4 +12,4 @@ metric_list:
   - metric: acc
   - metric: f1
 metadata:
-  - version: 1.0
+  version: 1.0
diff --git a/lm_eval/tasks/glue/rte/default.yaml b/lm_eval/tasks/glue/rte/default.yaml
@@ -11,4 +11,4 @@ doc_to_choice: ["True", "False"]
 metric_list:
   - metric: acc
 metadata:
-  - version: 1.0
+  version: 1.0
diff --git a/lm_eval/tasks/glue/sst2/default.yaml b/lm_eval/tasks/glue/sst2/default.yaml
@@ -11,4 +11,4 @@ doc_to_choice: ["negative", "positive"]
 metric_list:
   - metric: acc
 metadata:
-  - version: 1.0
+  version: 1.0
diff --git a/lm_eval/tasks/glue/wnli/default.yaml b/lm_eval/tasks/glue/wnli/default.yaml
@@ -11,4 +11,4 @@ doc_to_choice: ["False", "True"]
 metric_list:
   - metric: acc
 metadata:
-  - version: 2.0
+  version: 2.0
diff --git a/lm_eval/tasks/gsm8k/gsm8k-cot-self-consistency.yaml b/lm_eval/tasks/gsm8k/gsm8k-cot-self-consistency.yaml
@@ -31,4 +31,4 @@ filter_list:
       - function: "majority_vote"
       - function: "take_first"
 metadata:
-  - version: 0.0
+  version: 0.0
diff --git a/lm_eval/tasks/gsm8k/gsm8k-cot.yaml b/lm_eval/tasks/gsm8k/gsm8k-cot.yaml
@@ -41,4 +41,4 @@ filter_list:
         regex_pattern: "The answer is (\\-?[0-9\\.\\,]+)."
       - function: "take_first"
 metadata:
-  - version: 0.0
+  version: 0.0
diff --git a/lm_eval/tasks/gsm8k/gsm8k.yaml b/lm_eval/tasks/gsm8k/gsm8k.yaml
@@ -34,4 +34,4 @@ filter_list:
         regex_pattern: "#### (\\-?[0-9\\.\\,]+)"
       - function: "take_first"
 metadata:
-  - version: 1.0
+  version: 1.0
diff --git a/lm_eval/tasks/headqa/headqa_en.yaml b/lm_eval/tasks/headqa/headqa_en.yaml
@@ -20,4 +20,4 @@ metric_list:
     aggregation: mean
     higher_is_better: true
 metadata:
-  - version: 1.0
+  version: 1.0
diff --git a/lm_eval/tasks/hellaswag/hellaswag.yaml b/lm_eval/tasks/hellaswag/hellaswag.yaml
@@ -19,4 +19,4 @@ metric_list:
     aggregation: mean
     higher_is_better: true
 metadata:
-  - version: 1.0
+  version: 1.0
diff --git a/lm_eval/tasks/hendrycks_ethics/commonsense.yaml b/lm_eval/tasks/hendrycks_ethics/commonsense.yaml
@@ -12,4 +12,4 @@ doc_to_choice: ['no', 'yes']
 metric_list:
   - metric: acc
 metadata:
-  - version: 1.0
+  version: 1.0
diff --git a/lm_eval/tasks/hendrycks_ethics/deontology.yaml b/lm_eval/tasks/hendrycks_ethics/deontology.yaml
@@ -5,5 +5,5 @@ doc_to_text: "Question: Would most people believe this reasonable or unreasonabl
 doc_to_target: label
 doc_to_choice: ['unreasonable', 'reasonable']
 metadata:
-  - version: 1.0
+  version: 1.0
 # TODO: implement exact-match metric for this subset
diff --git a/lm_eval/tasks/hendrycks_ethics/justice.yaml b/lm_eval/tasks/hendrycks_ethics/justice.yaml
@@ -6,4 +6,4 @@ dataset_name: justice
 doc_to_text: "Question: Would most people believe this reasonable or unreasonable to say? \"{{scenario}}\"\nAnswer:"
 # TODO: impl. exact match for this and deontology
 metadata:
-  - version: 1.0
+  version: 1.0
diff --git a/lm_eval/tasks/hendrycks_ethics/utilitarianism.yaml b/lm_eval/tasks/hendrycks_ethics/utilitarianism.yaml
@@ -9,4 +9,4 @@ doc_to_choice: ['no', 'yes']
 metric_list:
   - metric: acc
 metadata:
-  - version: 1.0
+  version: 1.0
diff --git a/lm_eval/tasks/hendrycks_ethics/utilitarianism_original_yaml b/lm_eval/tasks/hendrycks_ethics/utilitarianism_original_yaml
@@ -13,4 +13,4 @@
 #   - metric: acc
 # TODO: we want this to be implemented as a winograd_schema task type, actually
 # metadata:
-#   - version: 1.0
+#   version: 1.0
diff --git a/lm_eval/tasks/hendrycks_ethics/virtue.yaml b/lm_eval/tasks/hendrycks_ethics/virtue.yaml
@@ -7,4 +7,4 @@ doc_to_text: "Sentence: {{scenario}}\nQuestion: Does the character in this sente
 doc_to_target: label
 doc_to_choice: ['no', 'yes']
 metadata:
-  - version: 1.0
+  version: 1.0
diff --git a/lm_eval/tasks/ifeval/ifeval.yaml b/lm_eval/tasks/ifeval/ifeval.yaml
@@ -26,4 +26,4 @@ metric_list:
     aggregation: !function utils.agg_inst_level_acc
     higher_is_better: true
 metadata:
-  - version: 1.0
+  version: 1.0
diff --git a/lm_eval/tasks/lambada/lambada_openai.yaml b/lm_eval/tasks/lambada/lambada_openai.yaml
@@ -17,4 +17,4 @@ metric_list:
     aggregation: mean
     higher_is_better: true
 metadata:
-  - version: 1.0
+  version: 1.0
diff --git a/lm_eval/tasks/lambada/lambada_standard.yaml b/lm_eval/tasks/lambada/lambada_standard.yaml
@@ -18,4 +18,4 @@ metric_list:
     aggregation: mean
     higher_is_better: true
 metadata:
-  - version: 1.0
+  version: 1.0
diff --git a/lm_eval/tasks/lambada_cloze/lambada_openai_cloze.yaml b/lm_eval/tasks/lambada_cloze/lambada_openai_cloze.yaml
@@ -17,4 +17,4 @@ metric_list:
     aggregation: mean
     higher_is_better: true
 metadata:
-  - version: 1.0
+  version: 1.0
diff --git a/lm_eval/tasks/lambada_cloze/lambada_standard_cloze.yaml b/lm_eval/tasks/lambada_cloze/lambada_standard_cloze.yaml
@@ -18,4 +18,4 @@ metric_list:
     aggregation: mean
     higher_is_better: true
 metadata:
-  - version: 1.0
+  version: 1.0
diff --git a/lm_eval/tasks/lambada_multilingual/lambada_mt_en.yaml b/lm_eval/tasks/lambada_multilingual/lambada_mt_en.yaml
@@ -17,4 +17,4 @@ metric_list:
     aggregation: mean
     higher_is_better: true
 metadata:
-  - version: 1.0
+  version: 1.0
diff --git a/lm_eval/tasks/logiqa/logiqa.yaml b/lm_eval/tasks/logiqa/logiqa.yaml
@@ -18,4 +18,4 @@ metric_list:
     aggregation: mean
     higher_is_better: true
 metadata:
-  - version: 1.0
+  version: 1.0
diff --git a/lm_eval/tasks/logiqa2/logieval.yaml b/lm_eval/tasks/logiqa2/logieval.yaml
@@ -24,4 +24,4 @@ filter_list:
         regex_pattern: "^\\s*([A-D])"
       - function: "take_first"
 metadata:
-  - version: 0.0
+  version: 0.0
diff --git a/lm_eval/tasks/logiqa2/logiqa2.yaml b/lm_eval/tasks/logiqa2/logiqa2.yaml
@@ -18,4 +18,4 @@ metric_list:
     aggregation: mean
     higher_is_better: true
 metadata:
-  - version: 0.0
+  version: 0.0
diff --git a/lm_eval/tasks/mathqa/mathqa.yaml b/lm_eval/tasks/mathqa/mathqa.yaml
@@ -19,4 +19,4 @@ metric_list:
     aggregation: mean
     higher_is_better: true
 metadata:
-  - version: 1.0
+  version: 1.0
diff --git a/lm_eval/tasks/mc_taco/default.yaml b/lm_eval/tasks/mc_taco/default.yaml
@@ -12,4 +12,4 @@ metric_list:
   - metric: acc
   - metric: f1
 metadata:
-  - version: 1.0
+  version: 1.0
diff --git a/lm_eval/tasks/mgsm/direct/direct_yaml b/lm_eval/tasks/mgsm/direct/direct_yaml
@@ -26,4 +26,4 @@ metric_list:
     ignore_case: true
     ignore_punctuation: true
 metadata:
-  - version: 0.0
+  version: 0.0
diff --git a/lm_eval/tasks/mgsm/en_cot/cot_yaml b/lm_eval/tasks/mgsm/en_cot/cot_yaml
@@ -28,4 +28,4 @@ filter_list:
         regex_pattern: "The answer is (\\-?[0-9\\.\\,]+)"
       - function: "take_first"
 metadata:
-  - version: 0.0
+  version: 0.0
diff --git a/lm_eval/tasks/mgsm/native_cot/cot_yaml b/lm_eval/tasks/mgsm/native_cot/cot_yaml
@@ -28,4 +28,4 @@ filter_list:
         regex_pattern: "The answer is (\\-?[0-9\\.\\,]+)"
       - function: "take_first"
 metadata:
-  - version: 1.0
+  version: 1.0
diff --git a/lm_eval/tasks/minerva_math/minerva_math_algebra.yaml b/lm_eval/tasks/minerva_math/minerva_math_algebra.yaml
@@ -21,4 +21,4 @@ metric_list:
     higher_is_better: true
 num_fewshot: 0
 metadata:
-  - version: 0.0
+  version: 0.0
diff --git a/lm_eval/tasks/mmlu/default/_default_template_yaml b/lm_eval/tasks/mmlu/default/_default_template_yaml
@@ -12,4 +12,4 @@ metric_list:
     aggregation: mean
     higher_is_better: true
 metadata:
-  - version: 0.0
+  version: 0.0