Add aggregate metrics in leaderboard tasks

huggingface · Nov 1, 2024 · 9e0a6d5 · 9e0a6d5
1 parent cc532e5
commit 9e0a6d5
Show file tree

Hide file tree

Showing 4 changed files with 16 additions and 0 deletions.
diff --git a/lm_eval/tasks/leaderboard/bbh_mc/_leaderboard_bbh.yaml b/lm_eval/tasks/leaderboard/bbh_mc/_leaderboard_bbh.yaml
@@ -24,3 +24,7 @@ task:
   - leaderboard_bbh_tracking_shuffled_objects_seven_objects
   - leaderboard_bbh_tracking_shuffled_objects_three_objects
   - leaderboard_bbh_web_of_lies
+aggregate_metric_list:
+  - metric: acc_norm
+    aggregation: mean
+    weight_by_size: true
diff --git a/lm_eval/tasks/leaderboard/gpqa/_leaderboard_gpqa.yaml b/lm_eval/tasks/leaderboard/gpqa/_leaderboard_gpqa.yaml
@@ -3,3 +3,7 @@ task:
   - leaderboard_gpqa_diamond
   - leaderboard_gpqa_extended
   - leaderboard_gpqa_main
+aggregate_metric_list:
+  - metric: acc_norm
+    aggregation: mean
+    weight_by_size: true
diff --git a/lm_eval/tasks/leaderboard/math/_leaderboard_math.yaml b/lm_eval/tasks/leaderboard/math/_leaderboard_math.yaml
@@ -7,3 +7,7 @@ task:
   - leaderboard_math_num_theory_hard
   - leaderboard_math_prealgebra_hard
   - leaderboard_math_precalculus_hard
+aggregate_metric_list:
+  - metric: exact_match
+    aggregation: mean
+    weight_by_size: true
diff --git a/lm_eval/tasks/leaderboard/musr/_musr.yaml b/lm_eval/tasks/leaderboard/musr/_musr.yaml
@@ -3,3 +3,7 @@ task:
   - leaderboard_musr_murder_mysteries
   - leaderboard_musr_object_placements
   - leaderboard_musr_team_allocation
+aggregate_metric_list:
+  - metric: acc_norm
+    aggregation: mean
+    weight_by_size: true