process_lm_eval_results.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Process the outputs of the lm eval harness\n",
    "\n",
    "Especially the MMLU because tasks are not averaged for us to achive whast shown on leaderboard"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "import os\n",
    "import json"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "# OUTPUT_DIR = \"/cmlscratch/jkirchen/adv-llm-root/lm-evaluation-harness/$OUTPUT_DIR\"\n",
    "# OUTPUT_DIR = \"/cmlscratch/jkirchen/adv-llm-root/output\"\n",
    "# OUTPUT_DIR = \"/cmlscratch/jkirchen/adv-llm-root/output/zero_shot\"\n",
    "OUTPUT_DIR = \"/cmlscratch/jkirchen/adv-llm-root/output/rand_smooth\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "def order_fn(s):\n",
    "    if \"vicuna\" in s:\n",
    "        if \"broken\" in s:\n",
    "            return 1\n",
    "        elif \"para\" in s:\n",
    "            return 2\n",
    "        elif \"filter\" in s:\n",
    "            return 3\n",
    "        else:\n",
    "            return 0\n",
    "    if \"falcon\" in s:\n",
    "        if \"broken\" in s:\n",
    "            return 5\n",
    "        elif \"para\" in s:\n",
    "            return 6\n",
    "        elif \"filter\" in s:\n",
    "            return 7\n",
    "        else:\n",
    "            return 4\n",
    "    if \"guanaco\" in s:\n",
    "        if \"broken\" in s:\n",
    "            return 9\n",
    "        elif \"para\" in s:\n",
    "            return 10\n",
    "        elif \"filter\" in s:\n",
    "            return 11\n",
    "        else:\n",
    "            return 8\n",
    "    if \"alpaca_7b_hf\" in s:\n",
    "        return 12\n",
    "    if \"alpaca_7b_w_smooth\" in s:\n",
    "        return 13\n",
    "    if \"openplatypus_7b_hf\" in s:\n",
    "        return 14\n",
    "    if \"openplatypus_7b_w_smooth\" in s:\n",
    "        return 15\n",
    "    if \"wizard70k_7b_hf\" in s:\n",
    "        return 16\n",
    "    if \"wizard70k_7b_w_smooth\" in s:\n",
    "        return 17\n",
    "    \n",
    "    if \"llama_Wizard70k_7b_baseline_HF\" in s:\n",
    "        return 18\n",
    "    if \"llama_Wizard70k_7b_randSmooth10_HF\" in s:\n",
    "        return 19\n",
    "\n",
    "    if \"llama2_Wizard70k_7b_baseline_updated_HF\" in s:\n",
    "        return 20\n",
    "    if \"llama2_Wizard70k_7b_RandNoise15_updated_HF\" in s:\n",
    "        return 21"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "/cmlscratch/jkirchen/adv-llm-root/output/rand_smooth/alpaca_7b_hf_arc_challenge.json\n",
      "/cmlscratch/jkirchen/adv-llm-root/output/rand_smooth/alpaca_7b_w_smooth_iam10_pl2_hf_arc_challenge.json\n",
      "/cmlscratch/jkirchen/adv-llm-root/output/rand_smooth/llama2_openplatypus_7b_hf_arc_challenge.json\n",
      "/cmlscratch/jkirchen/adv-llm-root/output/rand_smooth/llama2_openplatypus_7b_w_smooth_15_hf_arc_challenge.json\n",
      "/cmlscratch/jkirchen/adv-llm-root/output/rand_smooth/llama2_wizard70k_7b_hf_arc_challenge.json\n",
      "/cmlscratch/jkirchen/adv-llm-root/output/rand_smooth/llama2_wizard70k_7b_w_smooth_15_hf_arc_challenge.json\n",
      "/cmlscratch/jkirchen/adv-llm-root/output/rand_smooth/llama_Wizard70k_7b_baseline_HF_arc_challenge.json\n",
      "/cmlscratch/jkirchen/adv-llm-root/output/rand_smooth/llama_Wizard70k_7b_randSmooth10_HF_arc_challenge.json\n",
      "/cmlscratch/jkirchen/adv-llm-root/output/rand_smooth/llama2_Wizard70k_7b_baseline_updated_HF_arc_challenge.json\n",
      "/cmlscratch/jkirchen/adv-llm-root/output/rand_smooth/llama2_Wizard70k_7b_RandNoise15_updated_HF_arc_challenge.json\n",
      "{'alpaca_7b_hf_arc_challenge': [0.5426621160409556], 'alpaca_7b_w_smooth_iam10_pl2_hf_arc_challenge': [0.5511945392491467], 'llama2_openplatypus_7b_hf_arc_challenge': [0.5324232081911263], 'llama2_openplatypus_7b_w_smooth_15_hf_arc_challenge': [0.5187713310580204], 'llama2_wizard70k_7b_hf_arc_challenge': [0.5460750853242321], 'llama2_wizard70k_7b_w_smooth_15_hf_arc_challenge': [0.45051194539249145], 'llama_Wizard70k_7b_baseline_HF_arc_challenge': [0.5366894197952219], 'llama_Wizard70k_7b_randSmooth10_HF_arc_challenge': [0.5409556313993175], 'llama2_Wizard70k_7b_baseline_updated_HF_arc_challenge': [0.5494880546075085], 'llama2_Wizard70k_7b_RandNoise15_updated_HF_arc_challenge': [0.5401023890784983]}\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "{'alpaca_7b_hf_arc_challenge': 0.5426621160409556,\n",
       " 'alpaca_7b_w_smooth_iam10_pl2_hf_arc_challenge': 0.5511945392491467,\n",
       " 'llama2_openplatypus_7b_hf_arc_challenge': 0.5324232081911263,\n",
       " 'llama2_openplatypus_7b_w_smooth_15_hf_arc_challenge': 0.5187713310580204,\n",
       " 'llama2_wizard70k_7b_hf_arc_challenge': 0.5460750853242321,\n",
       " 'llama2_wizard70k_7b_w_smooth_15_hf_arc_challenge': 0.45051194539249145,\n",
       " 'llama_Wizard70k_7b_baseline_HF_arc_challenge': 0.5366894197952219,\n",
       " 'llama_Wizard70k_7b_randSmooth10_HF_arc_challenge': 0.5409556313993175,\n",
       " 'llama2_Wizard70k_7b_baseline_updated_HF_arc_challenge': 0.5494880546075085,\n",
       " 'llama2_Wizard70k_7b_RandNoise15_updated_HF_arc_challenge': 0.5401023890784983}"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Iterate over all files in OUTPUT_DIR (only top level)\n",
    "# same except this time and filter for rows that contain \"arc\" and that are files ending in .json\n",
    "target_metric = \"acc_norm\"\n",
    "top_level_files = [f for f in os.listdir(OUTPUT_DIR) if not os.path.isdir(f)]\n",
    "fn = lambda f: (\"arc\" in f and f.endswith(\".json\"))\n",
    "all_files = [f\"{OUTPUT_DIR}/{f}\" for f in top_level_files if fn(f)]\n",
    "\n",
    "all_files = sorted(all_files, key=order_fn)\n",
    "\n",
    "for f in all_files:\n",
    "    print(f)\n",
    "\n",
    "all_data = []\n",
    "for f in all_files:\n",
    "    with open(f, \"r\") as f:\n",
    "        data = json.load(f)\n",
    "        all_data.append(data)\n",
    "\n",
    "# print(json.dumps(all_data[0], indent=4))\n",
    "\n",
    "all_accuracies = {}\n",
    "\n",
    "for f,data in zip(all_files, all_data):\n",
    "    run_name = (f.split(\"/\")[-1]).split(\".\")[0]\n",
    "    per_task_metrics = data[\"results\"]\n",
    "\n",
    "    all_accs = [metrics[target_metric] for task,metrics in per_task_metrics.items()]\n",
    "    all_accuracies[run_name] = all_accs\n",
    "\n",
    "print(all_accuracies)\n",
    "\n",
    "# we don't need to \"average\" now just take first\n",
    "average_accuracies = {name:accs[0] for name,accs in all_accuracies.items()}\n",
    "\n",
    "average_accuracies"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "/cmlscratch/jkirchen/adv-llm-root/output/rand_smooth/alpaca_7b_hf_hellaswag.json\n",
      "/cmlscratch/jkirchen/adv-llm-root/output/rand_smooth/alpaca_7b_w_smooth_iam10_pl2_hf_hellaswag.json\n",
      "/cmlscratch/jkirchen/adv-llm-root/output/rand_smooth/llama2_openplatypus_7b_hf_hellaswag.json\n",
      "/cmlscratch/jkirchen/adv-llm-root/output/rand_smooth/llama2_openplatypus_7b_w_smooth_15_hf_hellaswag.json\n",
      "/cmlscratch/jkirchen/adv-llm-root/output/rand_smooth/llama2_wizard70k_7b_hf_hellaswag.json\n",
      "/cmlscratch/jkirchen/adv-llm-root/output/rand_smooth/llama2_wizard70k_7b_w_smooth_15_hf_hellaswag.json\n",
      "/cmlscratch/jkirchen/adv-llm-root/output/rand_smooth/llama_Wizard70k_7b_baseline_HF_hellaswag.json\n",
      "/cmlscratch/jkirchen/adv-llm-root/output/rand_smooth/llama_Wizard70k_7b_randSmooth10_HF_hellaswag.json\n",
      "/cmlscratch/jkirchen/adv-llm-root/output/rand_smooth/llama2_Wizard70k_7b_baseline_updated_HF_hellaswag.json\n",
      "/cmlscratch/jkirchen/adv-llm-root/output/rand_smooth/llama2_Wizard70k_7b_RandNoise15_updated_HF_hellaswag.json\n",
      "{'alpaca_7b_hf_hellaswag': [0.7903804023102967], 'alpaca_7b_w_smooth_iam10_pl2_hf_hellaswag': [0.7866958773152758], 'llama2_openplatypus_7b_hf_hellaswag': [0.80601473809998], 'llama2_openplatypus_7b_w_smooth_15_hf_hellaswag': [0.8015335590519816], 'llama2_wizard70k_7b_hf_hellaswag': [0.7949611631149174], 'llama2_wizard70k_7b_w_smooth_15_hf_hellaswag': [0.7849034056960765], 'llama_Wizard70k_7b_baseline_HF_hellaswag': [0.7789285002987453], 'llama_Wizard70k_7b_randSmooth10_HF_hellaswag': [0.7800238996215894], 'llama2_Wizard70k_7b_baseline_updated_HF_hellaswag': [0.7950607448715395], 'llama2_Wizard70k_7b_RandNoise15_updated_HF_hellaswag': [0.7917745469030074]}\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "{'alpaca_7b_hf_hellaswag': 0.7903804023102967,\n",
       " 'alpaca_7b_w_smooth_iam10_pl2_hf_hellaswag': 0.7866958773152758,\n",
       " 'llama2_openplatypus_7b_hf_hellaswag': 0.80601473809998,\n",
       " 'llama2_openplatypus_7b_w_smooth_15_hf_hellaswag': 0.8015335590519816,\n",
       " 'llama2_wizard70k_7b_hf_hellaswag': 0.7949611631149174,\n",
       " 'llama2_wizard70k_7b_w_smooth_15_hf_hellaswag': 0.7849034056960765,\n",
       " 'llama_Wizard70k_7b_baseline_HF_hellaswag': 0.7789285002987453,\n",
       " 'llama_Wizard70k_7b_randSmooth10_HF_hellaswag': 0.7800238996215894,\n",
       " 'llama2_Wizard70k_7b_baseline_updated_HF_hellaswag': 0.7950607448715395,\n",
       " 'llama2_Wizard70k_7b_RandNoise15_updated_HF_hellaswag': 0.7917745469030074}"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Iterate over all files in OUTPUT_DIR (only top level)\n",
    "# same except this time and filter for rows that contain \"truthfulqa\" and that are files ending in .json\n",
    "target_metric = \"acc_norm\"\n",
    "top_level_files = [f for f in os.listdir(OUTPUT_DIR) if not os.path.isdir(f)]\n",
    "fn = lambda f: (\"hellaswag\" in f and f.endswith(\".json\"))\n",
    "all_files = [f\"{OUTPUT_DIR}/{f}\" for f in top_level_files if fn(f)]\n",
    "\n",
    "all_files = sorted(all_files, key=order_fn)\n",
    "\n",
    "for f in all_files:\n",
    "    print(f)\n",
    "\n",
    "all_data = []\n",
    "for f in all_files:\n",
    "    with open(f, \"r\") as f:\n",
    "        data = json.load(f)\n",
    "        all_data.append(data)\n",
    "\n",
    "# print(json.dumps(all_data[0], indent=4))\n",
    "\n",
    "all_accuracies = {}\n",
    "\n",
    "for f,data in zip(all_files, all_data):\n",
    "    run_name = (f.split(\"/\")[-1]).split(\".\")[0]\n",
    "    per_task_metrics = data[\"results\"]\n",
    "\n",
    "    all_accs = [metrics[target_metric] for task,metrics in per_task_metrics.items()]\n",
    "    all_accuracies[run_name] = all_accs\n",
    "\n",
    "print(all_accuracies)\n",
    "\n",
    "# we don't need to \"average\" now just take first\n",
    "average_accuracies = {name:accs[0] for name,accs in all_accuracies.items()}\n",
    "\n",
    "average_accuracies"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "/cmlscratch/jkirchen/adv-llm-root/output/rand_smooth/alpaca_7b_hf_mmlu.json\n",
      "/cmlscratch/jkirchen/adv-llm-root/output/rand_smooth/alpaca_7b_w_smooth_iam10_pl2_hf_mmlu.json\n",
      "/cmlscratch/jkirchen/adv-llm-root/output/rand_smooth/llama2_openplatypus_7b_hf_mmlu.json\n",
      "/cmlscratch/jkirchen/adv-llm-root/output/rand_smooth/llama2_openplatypus_7b_w_smooth_15_hf_mmlu.json\n",
      "/cmlscratch/jkirchen/adv-llm-root/output/rand_smooth/llama2_wizard70k_7b_hf_mmlu.json\n",
      "/cmlscratch/jkirchen/adv-llm-root/output/rand_smooth/llama2_wizard70k_7b_w_smooth_15_hf_mmlu.json\n",
      "/cmlscratch/jkirchen/adv-llm-root/output/rand_smooth/llama_Wizard70k_7b_baseline_HF_mmlu.json\n",
      "/cmlscratch/jkirchen/adv-llm-root/output/rand_smooth/llama_Wizard70k_7b_randSmooth10_HF_mmlu.json\n",
      "/cmlscratch/jkirchen/adv-llm-root/output/rand_smooth/llama2_Wizard70k_7b_baseline_updated_HF_mmlu.json\n",
      "/cmlscratch/jkirchen/adv-llm-root/output/rand_smooth/llama2_Wizard70k_7b_RandNoise15_updated_HF_mmlu.json\n",
      "{'alpaca_7b_hf_mmlu': [0.27, 0.48148148148148145, 0.3815789473684211, 0.46, 0.47547169811320755, 0.4166666666666667, 0.3, 0.32, 0.3, 0.3236994219653179, 0.21568627450980393, 0.49, 0.40425531914893614, 0.2894736842105263, 0.33793103448275863, 0.29365079365079366, 0.2777777777777778, 0.33, 0.45806451612903226, 0.31527093596059114, 0.38, 0.5575757575757576, 0.5050505050505051, 0.5803108808290155, 0.36153846153846153, 0.26296296296296295, 0.37815126050420167, 0.2781456953642384, 0.5724770642201835, 0.32407407407407407, 0.5147058823529411, 0.5569620253164557, 0.48878923766816146, 0.3816793893129771, 0.6033057851239669, 0.5092592592592593, 0.4049079754601227, 0.3392857142857143, 0.5242718446601942, 0.6239316239316239, 0.47, 0.5593869731800766, 0.4393063583815029, 0.2424581005586592, 0.42483660130718953, 0.4533762057877814, 0.4567901234567901, 0.35106382978723405, 0.34028683181225555, 0.45955882352941174, 0.380718954248366, 0.4909090909090909, 0.3510204081632653, 0.4925373134328358, 0.54, 0.3614457831325301, 0.543859649122807], 'alpaca_7b_w_smooth_iam10_pl2_hf_mmlu': [0.28, 0.4666666666666667, 0.42105263157894735, 0.44, 0.4226415094339623, 0.3888888888888889, 0.27, 0.32, 0.27, 0.34104046242774566, 0.24509803921568626, 0.53, 0.4, 0.2631578947368421, 0.31724137931034485, 0.30423280423280424, 0.24603174603174602, 0.35, 0.3774193548387097, 0.29064039408866993, 0.35, 0.5333333333333333, 0.4898989898989899, 0.5854922279792746, 0.3384615384615385, 0.26296296296296295, 0.33613445378151263, 0.23841059602649006, 0.5431192660550459, 0.22685185185185186, 0.49019607843137253, 0.5527426160337553, 0.5201793721973094, 0.4122137404580153, 0.5950413223140496, 0.4722222222222222, 0.3803680981595092, 0.32142857142857145, 0.4854368932038835, 0.5769230769230769, 0.44, 0.541507024265645, 0.42196531791907516, 0.2424581005586592, 0.33986928104575165, 0.45016077170418006, 0.42901234567901236, 0.32978723404255317, 0.31029986962190353, 0.4338235294117647, 0.39215686274509803, 0.5, 0.3224489795918367, 0.42786069651741293, 0.57, 0.39156626506024095, 0.49707602339181284], 'llama2_openplatypus_7b_hf_mmlu': [0.22, 0.4222222222222222, 0.23684210526315788, 0.44, 0.4377358490566038, 0.4166666666666667, 0.21, 0.27, 0.23, 0.3352601156069364, 0.23529411764705882, 0.52, 0.33191489361702126, 0.2719298245614035, 0.25517241379310346, 0.2566137566137566, 0.2222222222222222, 0.15, 0.43870967741935485, 0.24630541871921183, 0.43, 0.593939393939394, 0.4494949494949495, 0.6476683937823834, 0.3641025641025641, 0.2518518518518518, 0.36554621848739494, 0.23841059602649006, 0.5376146788990825, 0.3287037037037037, 0.6372549019607843, 0.6582278481012658, 0.515695067264574, 0.42748091603053434, 0.5785123966942148, 0.5277777777777778, 0.5276073619631901, 0.32142857142857145, 0.3786407766990291, 0.6538461538461539, 0.41, 0.6155810983397191, 0.3959537572254335, 0.23798882681564246, 0.3954248366013072, 0.4887459807073955, 0.5061728395061729, 0.3333333333333333, 0.3644067796610169, 0.35661764705882354, 0.43137254901960786, 0.5545454545454546, 0.33877551020408164, 0.5572139303482587, 0.57, 0.37349397590361444, 0.5906432748538012], 'llama2_openplatypus_7b_w_smooth_15_hf_mmlu': [0.22, 0.34814814814814815, 0.18421052631578946, 0.39, 0.33962264150943394, 0.2916666666666667, 0.2, 0.22, 0.24, 0.2774566473988439, 0.24509803921568626, 0.42, 0.25957446808510637, 0.23684210526315788, 0.2689655172413793, 0.24338624338624337, 0.24603174603174602, 0.19, 0.36451612903225805, 0.2315270935960591, 0.38, 0.2909090909090909, 0.3939393939393939, 0.42487046632124353, 0.2717948717948718, 0.24074074074074073, 0.29831932773109243, 0.23841059602649006, 0.3467889908256881, 0.22685185185185186, 0.35784313725490197, 0.35864978902953587, 0.40358744394618834, 0.3435114503816794, 0.35537190082644626, 0.37037037037037035, 0.34355828220858897, 0.25892857142857145, 0.3592233009708738, 0.5299145299145299, 0.3, 0.48020434227330777, 0.35260115606936415, 0.23687150837988827, 0.3202614379084967, 0.3954983922829582, 0.36419753086419754, 0.31560283687943264, 0.2542372881355932, 0.21691176470588236, 0.31699346405228757, 0.3, 0.24489795918367346, 0.46766169154228854, 0.5, 0.2891566265060241, 0.47953216374269003], 'llama2_wizard70k_7b_hf_mmlu': [0.33, 0.42962962962962964, 0.4276315789473684, 0.48, 0.5018867924528302, 0.3888888888888889, 0.3, 0.39, 0.38, 0.4161849710982659, 0.20588235294117646, 0.57, 0.4085106382978723, 0.3157894736842105, 0.45517241379310347, 0.2751322751322751, 0.3492063492063492, 0.3, 0.5161290322580645, 0.3054187192118227, 0.41, 0.6121212121212121, 0.5858585858585859, 0.6683937823834197, 0.41025641025641024, 0.25555555555555554, 0.46218487394957986, 0.3509933774834437, 0.6128440366972477, 0.28703703703703703, 0.5931372549019608, 0.6497890295358649, 0.5605381165919282, 0.5190839694656488, 0.5537190082644629, 0.48148148148148145, 0.5214723926380368, 0.4107142857142857, 0.5631067961165048, 0.7008547008547008, 0.53, 0.6436781609195402, 0.5057803468208093, 0.2748603351955307, 0.4934640522875817, 0.5369774919614148, 0.5216049382716049, 0.38652482269503546, 0.35919165580182527, 0.5073529411764706, 0.4493464052287582, 0.5181818181818182, 0.44081632653061226, 0.5970149253731343, 0.65, 0.3674698795180723, 0.6666666666666666], 'llama2_wizard70k_7b_w_smooth_15_hf_mmlu': [0.27, 0.32592592592592595, 0.27631578947368424, 0.25, 0.35094339622641507, 0.2708333333333333, 0.31, 0.24, 0.32, 0.3352601156069364, 0.28431372549019607, 0.23, 0.2765957446808511, 0.2631578947368421, 0.4068965517241379, 0.2777777777777778, 0.2619047619047619, 0.25, 0.2709677419354839, 0.3103448275862069, 0.21, 0.24848484848484848, 0.3434343434343434, 0.3316062176165803, 0.3153846153846154, 0.2518518518518518, 0.25210084033613445, 0.33112582781456956, 0.3467889908256881, 0.3888888888888889, 0.24509803921568626, 0.270042194092827, 0.2825112107623318, 0.26717557251908397, 0.35537190082644626, 0.3055555555555556, 0.2822085889570552, 0.2767857142857143, 0.3786407766990291, 0.3247863247863248, 0.21, 0.41507024265644954, 0.25722543352601157, 0.2424581005586592, 0.30718954248366015, 0.2797427652733119, 0.2191358024691358, 0.2872340425531915, 0.2438070404172099, 0.31985294117647056, 0.238562091503268, 0.2818181818181818, 0.40816326530612246, 0.263681592039801, 0.3, 0.21084337349397592, 0.3567251461988304], 'llama_Wizard70k_7b_baseline_HF_mmlu': [0.3, 0.3851851851851852, 0.35526315789473684, 0.46, 0.4716981132075472, 0.3888888888888889, 0.25, 0.31, 0.28, 0.3352601156069364, 0.16666666666666666, 0.48, 0.39148936170212767, 0.2982456140350877, 0.3103448275862069, 0.29365079365079366, 0.2857142857142857, 0.32, 0.36129032258064514, 0.2660098522167488, 0.35, 0.4484848484848485, 0.4393939393939394, 0.46632124352331605, 0.3384615384615385, 0.24814814814814815, 0.33613445378151263, 0.31125827814569534, 0.45688073394495415, 0.24537037037037038, 0.47549019607843135, 0.47257383966244726, 0.4977578475336323, 0.37404580152671757, 0.5950413223140496, 0.4444444444444444, 0.4171779141104294, 0.36607142857142855, 0.39805825242718446, 0.5555555555555556, 0.4, 0.5223499361430396, 0.4161849710982659, 0.2424581005586592, 0.39215686274509803, 0.39228295819935693, 0.42592592592592593, 0.3404255319148936, 0.3070404172099087, 0.41911764705882354, 0.3954248366013072, 0.43636363636363634, 0.3346938775510204, 0.44776119402985076, 0.5, 0.3614457831325301, 0.543859649122807], 'llama_Wizard70k_7b_randSmooth10_HF_mmlu': [0.26, 0.3851851851851852, 0.35526315789473684, 0.45, 0.4641509433962264, 0.3958333333333333, 0.28, 0.27, 0.27, 0.3352601156069364, 0.19607843137254902, 0.49, 0.4127659574468085, 0.2719298245614035, 0.32413793103448274, 0.29894179894179895, 0.2857142857142857, 0.32, 0.36774193548387096, 0.2561576354679803, 0.37, 0.45454545454545453, 0.4696969696969697, 0.47668393782383417, 0.34615384615384615, 0.24074074074074073, 0.3445378151260504, 0.304635761589404, 0.47155963302752296, 0.25, 0.45588235294117646, 0.46835443037974683, 0.5022421524663677, 0.37404580152671757, 0.5867768595041323, 0.4537037037037037, 0.3987730061349693, 0.3392857142857143, 0.4077669902912621, 0.5683760683760684, 0.43, 0.5287356321839081, 0.41040462427745666, 0.2424581005586592, 0.3790849673202614, 0.40514469453376206, 0.4382716049382716, 0.35815602836879434, 0.3050847457627119, 0.41544117647058826, 0.3790849673202614, 0.44545454545454544, 0.3306122448979592, 0.4427860696517413, 0.51, 0.3614457831325301, 0.5321637426900585], 'llama2_Wizard70k_7b_baseline_updated_HF_mmlu': [0.3, 0.42962962962962964, 0.375, 0.53, 0.46037735849056605, 0.4444444444444444, 0.33, 0.4, 0.4, 0.3988439306358382, 0.22549019607843138, 0.55, 0.4085106382978723, 0.34210526315789475, 0.46206896551724136, 0.2777777777777778, 0.3492063492063492, 0.29, 0.5258064516129032, 0.3251231527093596, 0.43, 0.6, 0.5757575757575758, 0.6735751295336787, 0.43333333333333335, 0.24814814814814815, 0.44537815126050423, 0.33774834437086093, 0.6330275229357798, 0.3148148148148148, 0.5980392156862745, 0.6582278481012658, 0.57847533632287, 0.5343511450381679, 0.5702479338842975, 0.5185185185185185, 0.49693251533742333, 0.38392857142857145, 0.5728155339805825, 0.7136752136752137, 0.54, 0.6538952745849298, 0.5404624277456648, 0.2837988826815642, 0.47058823529411764, 0.5530546623794212, 0.5061728395061729, 0.3900709219858156, 0.3650586701434159, 0.5073529411764706, 0.4673202614379085, 0.509090909090909, 0.42857142857142855, 0.6268656716417911, 0.67, 0.4036144578313253, 0.6900584795321637], 'llama2_Wizard70k_7b_RandNoise15_updated_HF_mmlu': [0.31, 0.4222222222222222, 0.4276315789473684, 0.48, 0.47924528301886793, 0.4305555555555556, 0.33, 0.39, 0.38, 0.3815028901734104, 0.21568627450980393, 0.53, 0.3702127659574468, 0.3157894736842105, 0.43448275862068964, 0.2751322751322751, 0.3253968253968254, 0.29, 0.5193548387096775, 0.3251231527093596, 0.39, 0.6060606060606061, 0.5505050505050505, 0.6580310880829016, 0.43846153846153846, 0.26296296296296295, 0.42857142857142855, 0.304635761589404, 0.6110091743119266, 0.4027777777777778, 0.5931372549019608, 0.6751054852320675, 0.5605381165919282, 0.5190839694656488, 0.5867768595041323, 0.49074074074074076, 0.5398773006134969, 0.35714285714285715, 0.5339805825242718, 0.688034188034188, 0.5, 0.6168582375478927, 0.5202312138728323, 0.30614525139664805, 0.477124183006536, 0.5273311897106109, 0.5277777777777778, 0.3546099290780142, 0.3468057366362451, 0.5036764705882353, 0.46078431372549017, 0.5272727272727272, 0.4530612244897959, 0.6019900497512438, 0.65, 0.3795180722891566, 0.6549707602339181]}\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "{'alpaca_7b_hf_mmlu': 0.4148412104871215,\n",
       " 'alpaca_7b_w_smooth_iam10_pl2_hf_mmlu': 0.3981670400829601,\n",
       " 'llama2_openplatypus_7b_hf_mmlu': 0.40528005613309287,\n",
       " 'llama2_openplatypus_7b_w_smooth_15_hf_mmlu': 0.3165834779103289,\n",
       " 'llama2_wizard70k_7b_hf_mmlu': 0.46321992612417723,\n",
       " 'llama2_wizard70k_7b_w_smooth_15_hf_mmlu': 0.29176472719674396,\n",
       " 'llama_Wizard70k_7b_baseline_HF_mmlu': 0.38287488893104604,\n",
       " 'llama_Wizard70k_7b_randSmooth10_HF_mmlu': 0.38398685440903135,\n",
       " 'llama2_Wizard70k_7b_baseline_updated_HF_mmlu': 0.4692518433910401,\n",
       " 'llama2_Wizard70k_7b_RandNoise15_updated_HF_mmlu': 0.46031448728227586}"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Iterate over all files in OUTPUT_DIR (only top level thpugh)\n",
    "# and filter for rows that contain \"mmlu\" and that are files ending in .json\n",
    "target_metric = \"acc\"\n",
    "\n",
    "top_level_files = [f for f in os.listdir(OUTPUT_DIR) if not os.path.isdir(f)]\n",
    "fn = lambda f: (\"mmlu\" in f and f.endswith(\".json\"))\n",
    "all_files = [f\"{OUTPUT_DIR}/{f}\" for f in top_level_files if fn(f)]\n",
    "\n",
    "all_files = sorted(all_files, key=order_fn)\n",
    "\n",
    "for f in all_files:\n",
    "    print(f)\n",
    "\n",
    "all_data = []\n",
    "for f in all_files:\n",
    "    with open(f, \"r\") as f:\n",
    "        data = json.load(f)\n",
    "        all_data.append(data)\n",
    "\n",
    "# print(json.dumps(all_data[0], indent=4))\n",
    "\n",
    "all_accuracies = {}\n",
    "\n",
    "for f,data in zip(all_files, all_data):\n",
    "    run_name = (f.split(\"/\")[-1]).split(\".\")[0]\n",
    "    per_task_metrics = data[\"results\"]\n",
    "\n",
    "    all_accs = [metrics[target_metric] for task,metrics in per_task_metrics.items()]\n",
    "    all_accuracies[run_name] = all_accs\n",
    "\n",
    "print(all_accuracies)\n",
    "\n",
    "average_accuracies = {name:np.mean(accs) for name,accs in all_accuracies.items()}\n",
    "\n",
    "average_accuracies"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "/cmlscratch/jkirchen/adv-llm-root/output/rand_smooth/alpaca_7b_hf_truthfulqa.json\n",
      "/cmlscratch/jkirchen/adv-llm-root/output/rand_smooth/alpaca_7b_w_smooth_iam10_pl2_hf_truthfulqa.json\n",
      "/cmlscratch/jkirchen/adv-llm-root/output/rand_smooth/llama2_openplatypus_7b_hf_truthfulqa.json\n",
      "/cmlscratch/jkirchen/adv-llm-root/output/rand_smooth/llama2_openplatypus_7b_w_smooth_15_hf_truthfulqa.json\n",
      "/cmlscratch/jkirchen/adv-llm-root/output/rand_smooth/llama2_wizard70k_7b_hf_truthfulqa.json\n",
      "/cmlscratch/jkirchen/adv-llm-root/output/rand_smooth/llama2_wizard70k_7b_w_smooth_15_hf_truthfulqa.json\n",
      "/cmlscratch/jkirchen/adv-llm-root/output/rand_smooth/llama_Wizard70k_7b_baseline_HF_truthfulqa.json\n",
      "/cmlscratch/jkirchen/adv-llm-root/output/rand_smooth/llama_Wizard70k_7b_randSmooth10_HF_truthfulqa.json\n",
      "/cmlscratch/jkirchen/adv-llm-root/output/rand_smooth/llama2_Wizard70k_7b_baseline_updated_HF_truthfulqa.json\n",
      "/cmlscratch/jkirchen/adv-llm-root/output/rand_smooth/llama2_Wizard70k_7b_RandNoise15_updated_HF_truthfulqa.json\n",
      "{'alpaca_7b_hf_truthfulqa': [0.3887569694954727], 'alpaca_7b_w_smooth_iam10_pl2_hf_truthfulqa': [0.3900156301284775], 'llama2_openplatypus_7b_hf_truthfulqa': [0.4320702783461802], 'llama2_openplatypus_7b_w_smooth_15_hf_truthfulqa': [0.42477984441513983], 'llama2_wizard70k_7b_hf_truthfulqa': [0.4755360867696246], 'llama2_wizard70k_7b_w_smooth_15_hf_truthfulqa': [0.49880641232867035], 'llama_Wizard70k_7b_baseline_HF_truthfulqa': [0.4482897323462532], 'llama_Wizard70k_7b_randSmooth10_HF_truthfulqa': [0.454293414651063], 'llama2_Wizard70k_7b_baseline_updated_HF_truthfulqa': [0.4798857707339794], 'llama2_Wizard70k_7b_RandNoise15_updated_HF_truthfulqa': [0.47666477665746204]}\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "{'alpaca_7b_hf_truthfulqa': 0.3887569694954727,\n",
       " 'alpaca_7b_w_smooth_iam10_pl2_hf_truthfulqa': 0.3900156301284775,\n",
       " 'llama2_openplatypus_7b_hf_truthfulqa': 0.4320702783461802,\n",
       " 'llama2_openplatypus_7b_w_smooth_15_hf_truthfulqa': 0.42477984441513983,\n",
       " 'llama2_wizard70k_7b_hf_truthfulqa': 0.4755360867696246,\n",
       " 'llama2_wizard70k_7b_w_smooth_15_hf_truthfulqa': 0.49880641232867035,\n",
       " 'llama_Wizard70k_7b_baseline_HF_truthfulqa': 0.4482897323462532,\n",
       " 'llama_Wizard70k_7b_randSmooth10_HF_truthfulqa': 0.454293414651063,\n",
       " 'llama2_Wizard70k_7b_baseline_updated_HF_truthfulqa': 0.4798857707339794,\n",
       " 'llama2_Wizard70k_7b_RandNoise15_updated_HF_truthfulqa': 0.47666477665746204}"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Iterate over all files in OUTPUT_DIR (only top level)\n",
    "# same except this time and filter for rows that contain \"truthfulqa\" and that are files ending in .json\n",
    "target_metric = \"mc2\"\n",
    "top_level_files = [f for f in os.listdir(OUTPUT_DIR) if not os.path.isdir(f)]\n",
    "fn = lambda f: (\"truthfulqa\" in f and f.endswith(\".json\"))\n",
    "all_files = [f\"{OUTPUT_DIR}/{f}\" for f in top_level_files if fn(f)]\n",
    "\n",
    "all_files = sorted(all_files, key=order_fn)\n",
    "\n",
    "for f in all_files:\n",
    "    print(f)\n",
    "\n",
    "all_data = []\n",
    "for f in all_files:\n",
    "    with open(f, \"r\") as f:\n",
    "        data = json.load(f)\n",
    "        all_data.append(data)\n",
    "\n",
    "# print(json.dumps(all_data[0], indent=4))\n",
    "\n",
    "all_accuracies = {}\n",
    "\n",
    "for f,data in zip(all_files, all_data):\n",
    "    run_name = (f.split(\"/\")[-1]).split(\".\")[0]\n",
    "    per_task_metrics = data[\"results\"]\n",
    "\n",
    "    all_accs = [metrics[target_metric] for task,metrics in per_task_metrics.items()]\n",
    "    all_accuracies[run_name] = all_accs\n",
    "\n",
    "print(all_accuracies)\n",
    "\n",
    "# we don't need to \"average\" now just take first\n",
    "average_accuracies = {name:accs[0] for name,accs in all_accuracies.items()}\n",
    "\n",
    "average_accuracies"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "adv-llm-dev",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.12"
  },
  "orig_nbformat": 4
 },
 "nbformat": 4,
 "nbformat_minor": 2
}