CrossValidation_nback_allclasses_HR_only_with_ensemble.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "%matplotlib inline\n",
    "\n",
    "import os\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "import matplotlib.pyplot as plt\n",
    "import sklearn.datasets\n",
    "import sklearn.metrics\n",
    "from sklearn.inspection import plot_partial_dependence, permutation_importance\n",
    "\n",
    "import autosklearn.classification\n",
    "import autosklearn.metrics\n",
    "from sklearn.utils import shuffle\n",
    "from sklearn.model_selection import KFold"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "def collect_dataset_all_participants():\n",
    "    \n",
    "    data_test = pd.read_csv('df_SPSS_Final_all_VP_with_recording_problems_removed.csv')\n",
    "    data_test =  data_test.sort_values('VP').reset_index(drop=True)\n",
    "\n",
    "    appended_datapoints = data_test[['VP', 'baseline-eye_0_lhipa', 'baseline-eye_1_lhipa','baseline-HR', 'baseline-HRV', 'baseline-HR_std','baseline-HR_max', 'baseline-HR_min',\n",
    "                           'Nback_LL-eye_0_lhipa','Nback_LL-eye_1_lhipa','Nback_LL-HR', 'Nback_LL-HRV',  'Nback_LL-HR_std','Nback_LL-HR_max', 'Nback_LL-HR_min',\n",
    "                           'Nback_ML-eye_0_lhipa','Nback_ML-eye_1_lhipa', 'Nback_ML-HR','Nback_ML-HRV',  'Nback_ML-HR_std','Nback_ML-HR_max', 'Nback_ML-HR_min',\n",
    "                           'Nback_HL-eye_0_lhipa','Nback_HL-eye_1_lhipa', 'Nback_HL-HR','Nback_HL-HRV',  'Nback_HL-HR_std','Nback_HL-HR_max', 'Nback_HL-HR_min',\n",
    "                           'VIS_LL-eye_0_lhipa','VIS_LL-eye_1_lhipa', 'VIS_LL-HR', 'VIS_LL-HRV', 'VIS_LL-HR_std','VIS_LL-HR_max', 'VIS_LL-HR_min',\n",
    "                           'VIS_ML-eye_0_lhipa','VIS_ML-eye_1_lhipa', 'VIS_ML-HR', 'VIS_ML-HRV', 'VIS_ML-HR_std','VIS_ML-HR_max', 'VIS_ML-HR_min',\n",
    "                           'VIS_HL-eye_0_lhipa','VIS_HL-eye_1_lhipa', 'VIS_HL-HR', 'VIS_HL-HRV', 'VIS_HL-HR_std','VIS_HL-HR_max', 'VIS_HL-HR_min',]]\n",
    "    \n",
    "    ##### Labelling the data ####\n",
    "    \n",
    "    appended_datapoints_with_labels = []\n",
    "    number_of_features_per_class = 7\n",
    "    for i in range((len(appended_datapoints.columns)-1)//number_of_features_per_class):\n",
    "    #         print(i)\n",
    "        df_temp = appended_datapoints.iloc[:, np.r_[0, 1 + (i*number_of_features_per_class) : 1 + ((i+1)*number_of_features_per_class)]].copy()\n",
    "#         print(df_temp)\n",
    "        df_temp['label_task_class'] = appended_datapoints.columns[(i*number_of_features_per_class)+ 2 + 2].split('-')[0]\n",
    "#         print(df_temp)\n",
    "        if i == 0:\n",
    "            df_temp['label_mental_load_level'] = appended_datapoints.columns[(i*number_of_features_per_class)+ 2 + 1].split('-')[0]\n",
    "        else:\n",
    "            df_temp['label_mental_load_level'] = (appended_datapoints.columns[(i*number_of_features_per_class)+ 2 + 2].split('-')[0]).split('_')[1]\n",
    "#         print(df_temp)\n",
    "        df_temp.columns = ['Participant', appended_datapoints.columns[1].split('-')[1],appended_datapoints.columns[2].split('-')[1], \n",
    "                           appended_datapoints.columns[3].split('-')[1],appended_datapoints.columns[4].split('-')[1],\n",
    "                           appended_datapoints.columns[5].split('-')[1],appended_datapoints.columns[6].split('-')[1],\n",
    "                           appended_datapoints.columns[7].split('-')[1],\n",
    "                          'label_task_class','label_mental_load_level']\n",
    "#         print(df_temp)    \n",
    "\n",
    "        appended_datapoints_with_labels.append(df_temp)\n",
    "    appended_datapoints_with_labels = pd.concat(appended_datapoints_with_labels)\n",
    "    appended_datapoints_with_labels['label_task_class_factorized'] = pd.factorize(appended_datapoints_with_labels.label_task_class)[0]\n",
    "    appended_datapoints_with_labels['label_mental_load_level_factorized'] = pd.factorize(appended_datapoints_with_labels.label_mental_load_level)[0]\n",
    "    appended_datapoints_with_labels.reset_index(drop=True, inplace = True)\n",
    "    appended_datapoints_with_labels.sort_values(by=['Participant','label_task_class_factorized'], inplace=True)\n",
    "    appended_datapoints_with_labels.reset_index(drop=True, inplace = True)\n",
    "#     appended_datapoints_with_labels.to_csv('df_SPSS_Final_all_with_labels.csv',index =False)\n",
    "    \n",
    "    return appended_datapoints_with_labels"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "def class_type_to_number_of_classes(classes_type):\n",
    "    if classes_type == 'Mental_load_nback_only':\n",
    "            return 3\n",
    "    elif classes_type == 'Mental_load_and_Sec_task':\n",
    "            return 7\n",
    "    elif classes_type == 'Mental_load_two_levels':\n",
    "            return 2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "def Dataset_adjustor(no_classes):\n",
    "\n",
    "    #load dataset and put it in x and y\n",
    "    dataset__ = collect_dataset_all_participants()\n",
    "    if no_classes == 3:\n",
    "        # compare the two levels Nback_LL and Nback_ML (won't work with processed_ll)\n",
    "        dataset__ = dataset__[(dataset__['label_task_class'] == 'Nback_LL') | (dataset__['label_task_class'] == 'Nback_ML') | (dataset__['label_task_class'] == 'Nback_HL')].reset_index(drop=True)\n",
    "        dataset__['label_task_class_factorized'] = pd.factorize(dataset__.label_task_class)[0]\n",
    "        dataset__['label_mental_load_level_factorized'] = pd.factorize(dataset__.label_mental_load_level)[0]\n",
    "        y = dataset__['label_task_class_factorized'].values\n",
    "    \n",
    "    elif no_classes == 7:\n",
    "        y = dataset__['label_task_class_factorized'].values\n",
    "    \n",
    "    elif no_classes == 2:\n",
    "        \n",
    "        # compare the two levels Nback_LL and Nback_ML (won't work with processed_ll)\n",
    "        dataset__ = dataset__[(dataset__['label_task_class'] == 'Nback_LL') | (dataset__['label_task_class'] == 'Nback_ML')].reset_index(drop=True)\n",
    "        dataset__['label_task_class_factorized'] = pd.factorize(dataset__.label_task_class)[0]\n",
    "        dataset__['label_mental_load_level_factorized'] = pd.factorize(dataset__.label_mental_load_level)[0]\n",
    "        y = dataset__['label_task_class_factorized'].values\n",
    "\n",
    "    \n",
    "    x = dataset__[['HR','HRV','HR_std','HR_max','HR_min']].values.astype(float)\n",
    "        \n",
    "    return x , y"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "def main_inner_loop_cv(X_train, X_test, y_train, y_test):\n",
    "    \n",
    "    print(\"#\"*80)\n",
    "    # print(\"Use predefined accuracy metric\")\n",
    "    cls = autosklearn.classification.AutoSklearnClassifier(\n",
    "        time_left_for_this_task=360,\n",
    "        per_run_time_limit=30,\n",
    "        seed=1514,\n",
    "        metric=autosklearn.metrics.balanced_accuracy,\n",
    "#         ensemble_nbest=1,\n",
    "#         tmp_folder='/tmp/autosklearn_interpretable_models_example_tmp',\n",
    "#         include_estimators=['lda','adaboost','k_nearest_neighbors'],\n",
    "#         include_preprocessors=['no_preprocessing', 'polynomial', 'select_percentile_classification'],\n",
    "#         resampling_strategy='cv',\n",
    "#         resampling_strategy_arguments={'folds': 3},\n",
    "    )\n",
    "    cls.fit(X_train, y_train)\n",
    "\n",
    "\n",
    "    train_predictions = cls.predict(X_train)\n",
    "    train_accuracy_per_outer_fold = sklearn.metrics.accuracy_score(y_train, train_predictions)\n",
    "    print(\"Train Accuracy score\", train_accuracy_per_outer_fold)\n",
    "    print(f\"Train Classification report for classifier {cls}:\\n\" f\"{sklearn.metrics.classification_report(y_train, train_predictions)}\\n\")\n",
    "\n",
    "    predictions = cls.predict(X_test)\n",
    "    test_accuracy_per_outer_fold = sklearn.metrics.accuracy_score(y_test, predictions)\n",
    "    print(\"Test Accuracy score\", test_accuracy_per_outer_fold)\n",
    "    print(f\"Test Classification report for classifier {cls}:\\n\" f\"{sklearn.metrics.classification_report(y_test, predictions)}\\n\")\n",
    "    print(\"#\"*80)\n",
    "\n",
    "    print(cls.leaderboard())\n",
    "    print(\"#\"*80)\n",
    "\n",
    "    print(cls.sprint_statistics())\n",
    "    print(\"#\"*80)\n",
    "\n",
    "\n",
    "    features_name = ['HR','HRV','HR_std','HR_max','HR_min']\n",
    "\n",
    "    print('train permutation importance')\n",
    "\n",
    "    r = permutation_importance(cls, X_train, y_train,\n",
    "                               n_repeats=30,\n",
    "                               random_state=120)\n",
    "\n",
    "    sort_idx = r.importances_mean.argsort()[::-1]\n",
    "    plt.boxplot(r.importances[sort_idx].T, labels=[features_name[i] for i in sort_idx])\n",
    "    plt.xticks(rotation=90)\n",
    "    plt.tight_layout()\n",
    "    plt.show()\n",
    "\n",
    "    for i in sort_idx[::-1]:\n",
    "        print(f\"{features_name[i]:10s}: {r.importances_mean[i]:.3f} +/- \"\n",
    "              f\"{r.importances_std[i]:.3f}\")\n",
    "\n",
    "    print(\"#\"*80)\n",
    "\n",
    "    print('test permutation importance')\n",
    "\n",
    "    r = permutation_importance(cls, X_test, y_test,\n",
    "                               n_repeats=30,\n",
    "                               random_state=230)\n",
    "\n",
    "    sort_idx = r.importances_mean.argsort()[::-1]\n",
    "    plt.boxplot(r.importances[sort_idx].T, labels=[features_name[i] for i in sort_idx])\n",
    "    plt.xticks(rotation=90)\n",
    "    plt.tight_layout()\n",
    "    plt.show()\n",
    "\n",
    "    for i in sort_idx[::-1]:\n",
    "        print(f\"{features_name[i]:10s}: {r.importances_mean[i]:.3f} +/- \"\n",
    "              f\"{r.importances_std[i]:.3f}\")\n",
    "\n",
    "    print(\"#\"*80)\n",
    "    \n",
    "    return train_accuracy_per_outer_fold, test_accuracy_per_outer_fold"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "--------------------------------\n",
      "(135, 5) [[ 78.36956522  11.44072156   0.7330311   80.          77.        ]\n",
      " [ 80.67391304  13.95085566   2.04359389  86.          78.        ]\n",
      " [ 80.06521739  12.88111319   1.13064379  83.          78.        ]\n",
      " [ 95.34065934  22.34775654   2.64458717 102.          90.        ]\n",
      " [109.4673913   13.05188294   3.90498869 117.         100.        ]\n",
      " [105.8804348   16.1067865    5.0773204  118.          96.        ]\n",
      " [ 85.4673913   23.4605116    2.41575934  91.          81.        ]\n",
      " [ 90.34782609  18.33779499   2.71243025  97.          86.        ]\n",
      " [ 87.11956522  20.60379779   1.73732923  91.          84.        ]\n",
      " [ 76.65217391  49.6736603    5.23249625  90.          70.        ]\n",
      " [ 82.82417582  40.80835971   6.87341335  95.          75.        ]\n",
      " [ 86.66304348  38.77718826   2.98283923  93.          81.        ]\n",
      " [ 60.34482759 110.7744408    1.25787255  64.          58.        ]\n",
      " [ 66.23913043  69.55154623   1.8496155   69.          63.        ]\n",
      " [ 68.58695652  62.764167     2.29437959  72.          62.        ]\n",
      " [ 69.2826087   32.70161471   1.74658727  74.          67.        ]\n",
      " [ 73.54347826  18.00885008   3.4402131   79.          68.        ]\n",
      " [ 71.27173913  22.4102606    1.72010655  74.          69.        ]\n",
      " [ 57.62068966  50.90563545   0.86091877  60.          56.        ]\n",
      " [ 61.5         44.71620876   1.02469508  63.          59.        ]\n",
      " [ 61.57142857  44.99049282   1.58386686  65.          59.        ]\n",
      " [ 90.48351648  13.57121136   2.29841148  94.          84.        ]\n",
      " [ 84.70652174  20.83161165   4.54336124  95.          80.        ]\n",
      " [ 80.62365591  19.79020399   2.65554392  85.          75.        ]\n",
      " [ 93.40217391  20.94629449   3.33968165  99.          88.        ]\n",
      " [ 95.32608696  16.70691112   3.0435559  101.          89.        ]\n",
      " [103.          12.56630765   4.36388339 111.          95.        ]\n",
      " [ 61.17391304  29.02840594   1.57844651  64.          59.        ]\n",
      " [ 61.53846154  23.77487002   1.17956681  65.          59.        ]\n",
      " [ 63.23333333  28.00300947   2.20126226  68.          61.        ]\n",
      " [ 90.68478261  22.70063174   2.40438615  96.          86.        ]\n",
      " [102.9021739   24.72041466   5.73331186 111.          88.        ]\n",
      " [107.4565217   18.16166715   4.37982788 117.          99.        ]\n",
      " [ 80.65151515  25.24263932   2.09250049  85.          77.        ]\n",
      " [ 78.94656489  35.97392218   4.35331404  90.          74.        ]\n",
      " [ 80.78195489  30.7518169    4.03804901  92.          76.        ]\n",
      " [ 81.84946237  61.67463717   2.82822273  91.          78.        ]\n",
      " [ 85.48913043  44.65521148   4.33011338  94.          77.        ]\n",
      " [ 83.77173913  51.60809631   3.61790013  92.          78.        ]\n",
      " [ 72.09782609  52.50368904   1.60203527  75.          69.        ]\n",
      " [ 71.98484848  48.78007978   1.5762674   76.          69.        ]\n",
      " [ 71.28571429  47.23318414   1.17171362  74.          69.        ]\n",
      " [ 90.36956522  40.97908874   1.9433129   95.          86.        ]\n",
      " [ 94.67391304  32.9031212    2.82499979 101.          90.        ]\n",
      " [103.0108696   21.70329392   3.32479017 109.          97.        ]\n",
      " [ 62.57303371  40.5037877    1.16008175  65.          60.        ]\n",
      " [ 66.41304348  30.02654503   2.16770854  71.          64.        ]\n",
      " [ 71.41304348  29.74433181   5.19856204  80.          64.        ]\n",
      " [ 84.67391304  24.7373011    1.05406764  87.          83.        ]\n",
      " [103.8043478   20.95442464   5.89443203 114.          94.        ]\n",
      " [102.5326087   13.63254026   2.26720996 107.          97.        ]\n",
      " [ 83.84782609  39.3305239    1.56687714  87.          81.        ]\n",
      " [ 93.03296703  42.16080328   4.02587933 103.          87.        ]\n",
      " [ 98.65217391  30.24860364   3.9657084  109.          91.        ]\n",
      " [ 89.375       44.0356998    2.375       95.          85.        ]\n",
      " [ 97.55434783  36.97236865   4.39209797 109.          90.        ]\n",
      " [ 99.59340659  28.40950545   4.76474697 109.          91.        ]\n",
      " [ 71.07608696  57.74546269   2.63023265  76.          67.        ]\n",
      " [ 70.61538462  70.03919538   5.23254656  78.          62.        ]\n",
      " [ 68.93548387  58.80929016   3.78680917  75.          63.        ]\n",
      " [ 77.48913043  43.39532056   2.19928133  81.          72.        ]\n",
      " [ 81.60869565  35.20317793   2.28870821  87.          77.        ]\n",
      " [ 86.49450549  39.48135988   2.3222461   90.          80.        ]\n",
      " [ 77.48387097  26.61582563   0.91125464  79.          76.        ]\n",
      " [ 78.80434783  20.51855231   1.53395716  82.          76.        ]\n",
      " [ 82.48913043  18.35965476   1.85034994  86.          79.        ]\n",
      " [ 69.2826087   47.36809331   1.02497176  72.          68.        ]\n",
      " [ 78.86956522  44.63613576   3.02244283  85.          74.        ]\n",
      " [ 70.7311828   47.24910857   0.99611919  73.          69.        ]\n",
      " [ 84.9673913   20.54263858   1.49237982  89.          83.        ]\n",
      " [ 92.89247312  14.53818062   2.09177789  97.          88.        ]\n",
      " [ 87.3030303   19.2326059    2.94376432  93.          81.        ]\n",
      " [ 76.08695652  51.51464104   2.38497185  80.          70.        ]\n",
      " [ 82.84782609  30.89827336   4.15444964  89.          76.        ]\n",
      " [ 78.53763441  39.44368578   2.82675063  83.          72.        ]\n",
      " [ 64.52808989  69.04675623   1.52941119  68.          61.        ]\n",
      " [ 66.95652174  63.87221102   0.91974764  69.          65.        ]\n",
      " [ 67.65591398  52.80233855   1.21367321  69.          64.        ]\n",
      " [ 62.86021505  58.98341698   1.27499861  65.          60.        ]\n",
      " [ 66.04395604  49.20930355   2.90097402  73.          62.        ]\n",
      " [ 66.39130435  47.4524632    3.64440167  77.          63.        ]\n",
      " [ 61.2173913    5.24771012   0.412471    62.          61.        ]\n",
      " [ 62.02150538   5.00543183   0.29250474  63.          61.        ]\n",
      " [ 62.64516129   5.31711756   0.97985254  65.          62.        ]\n",
      " [ 76.36956522  36.9586939    1.5159829   79.          74.        ]\n",
      " [ 94.18478261  23.63667294   6.32443389 106.          84.        ]\n",
      " [ 90.72826087  25.52137652   4.66043304 102.          83.        ]\n",
      " [ 78.7173913   19.4427873    0.74200209  80.          77.        ]\n",
      " [ 82.94565217  20.50864197   2.92427669  89.          78.        ]\n",
      " [ 85.98924731  23.1467436    2.89045379  92.          83.        ]\n",
      " [ 69.80434783  63.07957647   3.18023492  78.          66.        ]\n",
      " [ 73.53846154  47.5688974    3.0645355   80.          70.        ]\n",
      " [ 70.15384615  50.31611184   3.66726858  76.          65.        ]\n",
      " [ 69.52688172  40.74843289   4.00427567  80.          65.        ]\n",
      " [ 75.4673913   37.95616326   3.3733324   81.          69.        ]\n",
      " [ 76.68478261  36.89069092   4.60842647  85.          70.        ]\n",
      " [ 69.98913043  26.25027472   1.2810879   72.          68.        ]\n",
      " [ 75.66666667  31.71013416   2.41114497  81.          72.        ]\n",
      " [ 85.25        31.36877428   5.14068385  95.          75.        ]\n",
      " [ 71.70652174  35.27848393   2.00292201  76.          69.        ]\n",
      " [ 82.04347826  21.06896056   2.78549443  89.          78.        ]\n",
      " [ 80.4673913   23.55400622   3.02328315  85.          75.        ]\n",
      " [ 70.88043478  33.24087911   1.79275348  74.          67.        ]\n",
      " [ 79.54347826  30.67446946   3.30155775  87.          75.        ]\n",
      " [ 71.8021978   34.66346139   2.79410596  79.          68.        ]\n",
      " [ 72.75        45.11328841   1.45680562  75.          70.        ]\n",
      " [ 76.83695652  34.95648787   1.6102739   80.          73.        ]\n",
      " [ 77.08695652  39.7682849    2.1551363   81.          73.        ]\n",
      " [ 75.94565217  22.17562269   4.28182237  82.          70.        ]\n",
      " [ 79.70652174  20.45391496   4.61457531  85.          71.        ]\n",
      " [ 79.06521739  16.86696492   2.7058887   83.          74.        ]\n",
      " [ 63.34831461  50.03260301   1.5147301   66.          60.        ]\n",
      " [ 64.90217391  38.89122616   1.72614034  69.          63.        ]\n",
      " [ 65.86956522  35.43311297   2.74344036  69.          58.        ]\n",
      " [ 97.52173913  36.20530591   2.19920075 102.          93.        ]\n",
      " [106.5054945   12.89142695   2.44667806 112.         101.        ]\n",
      " [111.4193548   10.76730155   2.47274424 117.         105.        ]\n",
      " [111.3548387   12.509996     2.51723163 116.         104.        ]\n",
      " [115.7717391   11.99587841   5.32447846 124.         104.        ]\n",
      " [120.5698925   10.38707391   6.27402539 135.         111.        ]\n",
      " [105.7717391   17.07900195   2.69904678 111.          98.        ]\n",
      " [108.0217391   18.0323275    3.48280953 117.         101.        ]\n",
      " [104.6593407   15.2894299    2.22976754 110.         100.        ]\n",
      " [ 82.05434783  51.03101836   1.28844472  85.          79.        ]\n",
      " [ 90.98913043 121.5687297    5.68081165 102.          81.        ]\n",
      " [ 88.37634409  30.56194707   1.76534123  93.          85.        ]\n",
      " [ 60.25555556  53.29523452   0.86359856  62.          59.        ]\n",
      " [ 65.75        40.15957182   3.04896632  71.          59.        ]\n",
      " [ 67.7173913   40.05970818   2.77146205  71.          62.        ]\n",
      " [ 68.47191011  55.46733273   3.69357136  76.          62.        ]\n",
      " [ 67.08860759  63.07616479   1.97573811  73.          64.        ]\n",
      " [ 69.42045455  43.99242359   1.42798007  73.          67.        ]\n",
      " [ 93.5         18.73675356   1.99183114  98.          90.        ]\n",
      " [ 93.66666667  17.51710128   2.4712911   99.          88.        ]\n",
      " [ 90.80645161  18.25573168   3.7019555   99.          85.        ]] (135,) [0 1 2 0 1 2 0 1 2 0 1 2 0 1 2 0 1 2 0 1 2 0 1 2 0 1 2 0 1 2 0 1 2 0 1 2 0\n",
      " 1 2 0 1 2 0 1 2 0 1 2 0 1 2 0 1 2 0 1 2 0 1 2 0 1 2 0 1 2 0 1 2 0 1 2 0 1\n",
      " 2 0 1 2 0 1 2 0 1 2 0 1 2 0 1 2 0 1 2 0 1 2 0 1 2 0 1 2 0 1 2 0 1 2 0 1 2\n",
      " 0 1 2 0 1 2 0 1 2 0 1 2 0 1 2 0 1 2 0 1 2 0 1 2]\n"
     ]
    }
   ],
   "source": [
    "#features options\n",
    "classes_type = 'Mental_load_nback_only' #options: 'Mental_load' for independency from tasks (4 classes) or 'Mental_load_and_Sec_task' for relating to all task levels (7 classes) or 'Mental_load_two_levels' for testing which uses 2 classes 'Nback_ML' and 'Nback_HL'\n",
    "no_classes = class_type_to_number_of_classes(classes_type)\n",
    "k_folds = 5\n",
    "\n",
    "\n",
    "\n",
    "x_all , y_all = Dataset_adjustor(no_classes)\n",
    "# x_all , y_all = shuffle(x_all , y_all)\n",
    "# Define the K-fold Cross Validator\n",
    "kfold = KFold(n_splits=k_folds)\n",
    "\n",
    "# Start print\n",
    "print('--------------------------------')\n",
    "print(x_all.shape, x_all , y_all.shape , y_all)\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "-=========================================-\n",
      "FOLD 0\n",
      "------------\n",
      "[ 27  28  29  30  31  32  33  34  35  36  37  38  39  40  41  42  43  44\n",
      "  45  46  47  48  49  50  51  52  53  54  55  56  57  58  59  60  61  62\n",
      "  63  64  65  66  67  68  69  70  71  72  73  74  75  76  77  78  79  80\n",
      "  81  82  83  84  85  86  87  88  89  90  91  92  93  94  95  96  97  98\n",
      "  99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116\n",
      " 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134] [ 0  1  2  3  4  5  6  7  8  9 10 11 12 13 14 15 16 17 18 19 20 21 22 23\n",
      " 24 25 26]\n",
      "108 27\n",
      "(108, 5) (108,)\n",
      "(27, 5) (27,)\n",
      "################################################################################\n"
     ]
    }
   ],
   "source": [
    "#X_train, X_test, y_train, y_test = sklearn.model_selection.train_test_split(x_all, y_all)\n",
    "#print(X_train.shape, X_test.shape, y_train.shape, y_test.shape)\n",
    "\n",
    "# For fold results\n",
    "train_accuracy_average = []\n",
    "test_accuracy_average = []\n",
    "\n",
    "########### for debugging ##########\n",
    "for fold, (train_ids, test_ids) in enumerate(kfold.split(np.arange(len(x_all)))):\n",
    "    \n",
    "    print('-=========================================-')\n",
    "    print(f'FOLD {fold}')\n",
    "    print('------------')\n",
    "    print(train_ids, test_ids)\n",
    "    print(len(train_ids), len(test_ids))\n",
    "#     print(x_all[test_ids])\n",
    "#     print(y_all[test_ids])\n",
    "    \n",
    "    X_train = x_all[train_ids]\n",
    "    y_train = y_all[train_ids]\n",
    "    \n",
    "    X_test = x_all[test_ids]\n",
    "    y_test = y_all[test_ids]\n",
    "\n",
    "    print(X_train.shape, y_train.shape)\n",
    "    print(X_test.shape, y_test.shape)\n",
    "    \n",
    "    train_accuracy, test_accuracy = main_inner_loop_cv(X_train, X_test, y_train, y_test)\n",
    "    train_accuracy_average.append(train_accuracy)\n",
    "    test_accuracy_average.append(test_accuracy)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "print(train_accuracy_average,test_accuracy_average)\n",
    "print(np.mean(train_accuracy_average), np.mean(test_accuracy_average))\n",
    "print(\"#\"*80)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}