Riemannian Models, Exploration, Tuning 2.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Final validation of selected models and prediction on 3rd session for all subjects\n",
    "import mne\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "import sklearn\n",
    "from sklearn import svm \n",
    "from sklearn.svm import SVC \n",
    "from sklearn.svm import NuSVC\n",
    "from sklearn.pipeline import Pipeline\n",
    "from sklearn.metrics import accuracy_score\n",
    "from sklearn.pipeline import make_pipeline\n",
    "from sklearn.preprocessing import StandardScaler\n",
    "from sklearn.model_selection import train_test_split\n",
    "from sklearn.linear_model import LogisticRegression\n",
    "from sklearn.ensemble import VotingClassifier as VC\n",
    "from sklearn.neighbors import KNeighborsClassifier as sknn\n",
    "from sklearn.metrics import confusion_matrix as cm\n",
    "from sklearn.ensemble import StackingClassifier as SC\n",
    "\n",
    "from mne.externals.pymatreader import read_mat\n",
    "\n",
    "from pyriemann.utils.distance import distance_riemann\n",
    "from pyriemann.classification import MDM\n",
    "from pyriemann.classification import KNearestNeighbor as knn\n",
    "from pyriemann.classification import TSclassifier as tsc\n",
    "from pyriemann.classification import FgMDM\n",
    "from pyriemann.estimation import Covariances\n",
    "from pyriemann.estimation import (XdawnCovariances, HankelCovariances,\n",
    "                                  CospCovariances, ERPCovariances)\n",
    "from pyriemann.spatialfilters import Xdawn, CSP\n",
    "from pyriemann.tangentspace import TangentSpace\n",
    "\n",
    "\n",
    "import xgboost as xgb\n",
    "\n",
    "import matplotlib.pyplot as plt\n",
    "%matplotlib qt\n",
    "\n",
    "import pickle\n",
    "import joblib\n",
    "import os\n",
    "import time\n",
    "from collections import OrderedDict\n",
    "\n",
    "mne.set_log_level('ERROR')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "def load_dat(sub_n,session_n,epf=False):\n",
    "    cwd = os.getcwd()\n",
    "    data_path = cwd\n",
    "    diff = ['MATBeasy', 'MATBmed', 'MATBdiff']\n",
    "\n",
    "    epochs_data = []\n",
    "    labels = []\n",
    "    for lab_idx, level in enumerate(diff):\n",
    "        sub = 'P{0:02d}'.format(sub_n)\n",
    "        sess = f'S{session_n}'\n",
    "        path = os.path.join(os.path.join(data_path, sub), sess) + f'/eeg/alldata_sbj{str(sub_n).zfill(2)}_sess{session_n}_{level}.set'\n",
    "        # Read the epoched data with MNE\n",
    "        epochs = mne.io.read_epochs_eeglab(path, verbose=False)\n",
    "        # You could add some pre-processing here with MNE\n",
    "\n",
    "        # Get the data and concatenante with others MATB levels\n",
    "        tmp = epochs.get_data()\n",
    "        epochs_data.extend(tmp)\n",
    "        labels.extend([lab_idx]*len(tmp))\n",
    "    epochs_datan = np.array(epochs_data)\n",
    "    labels = np.array(labels)\n",
    "    if epf ==False:\n",
    "        return epochs_datan, labels\n",
    "    if epf==True:\n",
    "        return epochs_datan, labels, epochs_data\n",
    "    \n",
    "def load_test_dat(sub_n,session_n=3,epf=False):\n",
    "    cwd = os.getcwd()\n",
    "    data_path = cwd\n",
    "    epochs_data = []\n",
    "\n",
    "    sub = 'P{0:02d}'.format(sub_n)\n",
    "    sess = f'S{session_n}'\n",
    "    path = os.path.join(os.path.join(data_path, sub), sess) + f'/eeg/testset_sbj{str(sub_n).zfill(2)}_sess{session_n}.set'\n",
    "    # Read the epoched data with MNE\n",
    "    epochs = mne.io.read_epochs_eeglab(path, verbose=False)\n",
    "    tmp = epochs.get_data()\n",
    "    epochs_data.extend(tmp)\n",
    "    epochs_datan = np.array(epochs_data)\n",
    "\n",
    "    if epf ==False:\n",
    "        return epochs_datan\n",
    "    if epf==True:\n",
    "        return epochs_datan, epochs_data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "jupyter": {
     "source_hidden": true
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "# ina2 = make_pipeline(HankelCovariances(delays=[2, 4, 8, 12, 16], estimator='oas'),\n",
    "#                     TangentSpace(metric='logeuclid'),\n",
    "#                     NuSVC())\n",
    "# ina2.fit(X2,y2)\n",
    "# p1 = ina2.predict(X1)\n",
    "# p2 = ina2.predict(X2)\n",
    "# print(\"tsc-svc hard train = \",accuracy_score(y2,p2),\" test = \",accuracy_score(y1,p1))\n",
    "\n",
    "# c = np.around(cm(y1,p1,normalize='true'),2)\n",
    "# c"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "# For the performance evaluation. Concatenate the session 1 and 2, \n",
    "# do train-test split (stratify class and group i.e. session) 80-20\n",
    "# Do CV on the training data - See mean accuracy\n",
    "# But I don't think this is a good method if data from 2 different sessions may be there, tsupdate won't work properly\n",
    "\n",
    "mdmc = make_pipeline(Covariances(estimator='oas'),MDM(n_jobs=-1))\n",
    "Fgmdmc = make_pipeline(Covariances(estimator='oas'),FgMDM(n_jobs=-1))\n",
    "\n",
    "tsck2 = make_pipeline(Covariances(estimator='oas'),tsc(tsupdate=True,clf=sknn(n_neighbors = 14,n_jobs=-1,weights='distance')))\n",
    "svock2 = VC(estimators = [('mdm',mdmc),('Fgmdm',Fgmdmc),('tsc',tsck2)], voting='hard',n_jobs=-1)\n",
    "\n",
    "tscs3 = make_pipeline(Covariances(estimator='oas'),tsc(tsupdate=True,clf=NuSVC()))\n",
    "svocs3 = VC(estimators = [('mdm',mdmc),('Fgmdm',Fgmdmc),('tsc',tscs3)], voting='hard',n_jobs=-1)\n",
    "\n",
    "tscs2 = make_pipeline(Covariances(estimator='oas'),tsc(tsupdate=True,clf=SVC(gamma='auto')))\n",
    "svocs2 = VC(estimators = [('mdm',mdmc),('Fgmdm',Fgmdmc),('tsc',tscs2)], voting='hard',n_jobs=-1)\n",
    "\n",
    "tscx = make_pipeline(Covariances(estimator='oas'),tsc(tsupdate=True,clf=xgb.XGBClassifier(n_estimators = 1500,max_depth = 15,learning_rate= 0.3,reg_lambda = 5,n_jobs=-1)))\n",
    "# tscs2 = make_pipeline(Covariances(estimator='oas'),tsc(tsupdate=True,clf=SVC(gamma='auto')))\n",
    "sf1 = VC(estimators = [('tsc',tscs2),('tscx',tscx),('mdm',mdmc)],weights = [1,1,2], voting='hard',n_jobs=-1)\n",
    "\n",
    "tscs3s = make_pipeline(Covariances(estimator='oas'),tsc(tsupdate=True,clf=NuSVC(probability=True)))\n",
    "\n",
    "## The submitted model\n",
    "ms = VC(estimators = [('mdm',mdmc),('tscx',tscx),('NuSVC',tscs3s)],voting='soft',n_jobs=-1)\n",
    "\n",
    "array_clfs = {}\n",
    "array_clfs[\"sknn_hard\"] = svock2\n",
    "array_clfs[\"NuSVC_hard\"] = svocs3\n",
    "array_clfs[\"NuSVC_xgb_soft\"] = ms\n",
    "array_clfs[\"xgb_SVC_weighted_hard\"] = sf1\n",
    "array_clfs[\"SVC_hard\"] = svocs2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(447, 61, 500) (447,) (447, 61, 500) (447,) (447, 61, 500)\n"
     ]
    }
   ],
   "source": [
    "# Load a subject, run all classifier, delete data variables, repeat\n",
    "\n",
    "X_test,y_test = load_dat(1,1)\n",
    "X_train,y_train = load_dat(1,2)\n",
    "X_p = load_test_dat(1,3)\n",
    "print(X_test.shape,y_test.shape,X_train.shape,y_train.shape,X_p.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'D:/Softwares/AnacondaWorkDir/Neuroergo_Hack/Predictions/sknn_hard_1.csv'"
      ]
     },
     "execution_count": 21,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "sub = 1\n",
    "fname = f\"D:/Softwares/AnacondaWorkDir/Neuroergo_Hack/Predictions/{list(array_clfs.keys())[0]}_{sub}.csv\"\n",
    "fname"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "model sknn_hard,Train time = 13.102108478546143, Predict time = 50.78337216377258, train acc = 1.0, test acc = 0.6554809843400448.\n",
      "[[0.87 0.03 0.1 ]\n",
      " [0.05 0.85 0.11]\n",
      " [0.19 0.56 0.25]]\n",
      "model NuSVC_hard,Train time = 11.35181736946106, Predict time = 51.6494517326355, train acc = 1.0, test acc = 0.6756152125279642.\n",
      "[[0.84 0.07 0.09]\n",
      " [0.06 0.88 0.06]\n",
      " [0.16 0.53 0.31]]\n",
      "model NuSVC_xgb_soft,Train time = 27.175652027130127, Predict time = 93.99026417732239, train acc = 1.0, test acc = 0.6621923937360179.\n",
      "[[0.7  0.23 0.07]\n",
      " [0.09 0.66 0.26]\n",
      " [0.11 0.26 0.63]]\n",
      "model xgb_SVC_weighted_hard,Train time = 25.153958082199097, Predict time = 98.27040338516235, train acc = 0.8747203579418344, test acc = 0.6599552572706935.\n",
      "[[0.77 0.19 0.04]\n",
      " [0.16 0.62 0.21]\n",
      " [0.11 0.3  0.59]]\n",
      "model SVC_hard,Train time = 11.288834571838379, Predict time = 50.26536464691162, train acc = 0.901565995525727, test acc = 0.7002237136465325.\n",
      "[[0.88 0.03 0.09]\n",
      " [0.05 0.9  0.05]\n",
      " [0.12 0.56 0.32]]\n"
     ]
    }
   ],
   "source": [
    "X_test,y_test = load_dat(1,1)\n",
    "X_train,y_train = load_dat(1,2)\n",
    "X_p = load_test_dat(1,3)\n",
    "\n",
    "for key,clf in array_clfs.items():\n",
    "    sub = 1\n",
    "    fname = f\"D:/Softwares/AnacondaWorkDir/Neuroergo_Hack/Predictions/{key}_{sub}.csv\"\n",
    "    \n",
    "    start = time.time()\n",
    "    clf.fit(X_train,y_train)\n",
    "    end = time.time()\n",
    "    t1 = end-start\n",
    "    \n",
    "    start = time.time()\n",
    "    p_train = clf.predict(X_train)\n",
    "    p_test = clf.predict(X_test)\n",
    "    end = time.time()\n",
    "    t2 = end-start\n",
    "    \n",
    "    c = np.around(cm(y_test,p_test,normalize='true'),2)\n",
    "    \n",
    "    p_pred = clf.predict(X_p)\n",
    "    pd.DataFrame(p_pred).to_csv(fname)\n",
    "    \n",
    "    print(f'model {key},Train time = {t1}, Predict time = {t2}, train acc = {accuracy_score(y_train,p_train)}, test acc = {accuracy_score(y_test,p_test)}.')\n",
    "    print(c)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "import pickle\n",
    "for key,clf in array_clfs.items():\n",
    "    sub = 1\n",
    "    filename = f\"D:/Softwares/AnacondaWorkDir/Neuroergo_Hack/Predictions/{key}_{sub}.pickle\"\n",
    "    pickle.dump(clf, open(filename, 'wb'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true,
    "jupyter": {
     "outputs_hidden": true
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "model sknn_hard,Train time = 55.83969163894653, Predict time = 70.01193857192993, train acc = 1.0, test acc = 0.6554809843400448.\n",
      "[[0.87 0.03 0.1 ]\n",
      " [0.05 0.85 0.11]\n",
      " [0.19 0.56 0.25]]\n",
      "model NuSVC_hard,Train time = 13.053358793258667, Predict time = 55.336960554122925, train acc = 1.0, test acc = 0.6756152125279642.\n",
      "[[0.84 0.07 0.09]\n",
      " [0.06 0.88 0.06]\n",
      " [0.16 0.53 0.31]]\n",
      "model NuSVC_xgb_soft,Train time = 37.33266758918762, Predict time = 90.49095058441162, train acc = 1.0, test acc = 0.6599552572706935.\n",
      "[[0.69 0.23 0.08]\n",
      " [0.09 0.66 0.25]\n",
      " [0.11 0.26 0.62]]\n",
      "model xgb_SVC_weighted_hard,Train time = 46.777177572250366, Predict time = 91.1038875579834, train acc = 0.8747203579418344, test acc = 0.6599552572706935.\n",
      "[[0.77 0.19 0.04]\n",
      " [0.16 0.62 0.21]\n",
      " [0.11 0.3  0.59]]\n"
     ]
    }
   ],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {
    "collapsed": true,
    "jupyter": {
     "outputs_hidden": true,
     "source_hidden": true
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "model SVC_hard,Train time = 20.383556127548218, Predict time = 49.5918607711792, train acc = 0.901565995525727, test acc = 0.7002237136465325.\n",
      "[[0.88 0.03 0.09]\n",
      " [0.05 0.9  0.05]\n",
      " [0.12 0.56 0.32]]\n"
     ]
    }
   ],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {
    "collapsed": true,
    "jupyter": {
     "outputs_hidden": true,
     "source_hidden": true
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "None\n",
      "None\n"
     ]
    }
   ],
   "source": [
    "from sklearn.utils.validation import check_is_fitted\n",
    "\n",
    "a = check_is_fitted(array_clfs[\"sknn_hard\"])\n",
    "print(a)\n",
    "a = check_is_fitted(svock2)\n",
    "print(a)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {
    "jupyter": {
     "source_hidden": true
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2\n",
      "model sknn_hard,Train time = 10.802088737487793, Predict time = 56.60003852844238, train acc = 1.0, test acc = 0.5503355704697986.\n",
      "[[0.77 0.01 0.22]\n",
      " [0.17 0.01 0.83]\n",
      " [0.11 0.01 0.87]]\n",
      "model NuSVC_hard,Train time = 13.638078212738037, Predict time = 55.22952437400818, train acc = 1.0, test acc = 0.5704697986577181.\n",
      "[[0.78 0.01 0.21]\n",
      " [0.15 0.01 0.85]\n",
      " [0.06 0.01 0.93]]\n",
      "model NuSVC_xgb_soft,Train time = 23.57753300666809, Predict time = 94.15355610847473, train acc = 1.0, test acc = 0.5838926174496645.\n",
      "[[0.81 0.03 0.15]\n",
      " [0.19 0.11 0.7 ]\n",
      " [0.11 0.05 0.83]]\n",
      "model xgb_SVC_weighted_hard,Train time = 22.97852897644043, Predict time = 92.6718316078186, train acc = 0.7829977628635347, test acc = 0.5458612975391499.\n",
      "[[0.79 0.05 0.16]\n",
      " [0.21 0.08 0.7 ]\n",
      " [0.15 0.08 0.77]]\n",
      "model SVC_hard,Train time = 11.410377740859985, Predict time = 51.59993624687195, train acc = 0.8568232662192393, test acc = 0.5659955257270693.\n",
      "[[0.79 0.01 0.21]\n",
      " [0.16 0.01 0.83]\n",
      " [0.08 0.01 0.91]]\n"
     ]
    }
   ],
   "source": [
    "sub = 2\n",
    "X_test,y_test = load_dat(sub,1)\n",
    "X_train,y_train = load_dat(sub,2)\n",
    "X_p = load_test_dat(sub,3)\n",
    "print(sub)\n",
    "for key,clf in array_clfs.items():\n",
    "    sub = 2\n",
    "    fname = f\"D:/Softwares/AnacondaWorkDir/Neuroergo_Hack/Predictions/{key}_{sub}.csv\"\n",
    "    \n",
    "    start = time.time()\n",
    "    clf.fit(X_train,y_train)\n",
    "    end = time.time()\n",
    "    t1 = end-start\n",
    "    \n",
    "    start = time.time()\n",
    "    p_train = clf.predict(X_train)\n",
    "    p_test = clf.predict(X_test)\n",
    "    end = time.time()\n",
    "    t2 = end-start\n",
    "    \n",
    "    c = np.around(cm(y_test,p_test,normalize='true'),2)\n",
    "    \n",
    "    p_pred = clf.predict(X_p)\n",
    "    pd.DataFrame(p_pred).to_csv(fname)\n",
    "    \n",
    "    print(f'model {key},Train time = {t1}, Predict time = {t2}, train acc = {accuracy_score(y_train,p_train)}, test acc = {accuracy_score(y_test,p_test)}.')\n",
    "    print(c)\n",
    "    \n",
    "    filename = f\"D:/Softwares/AnacondaWorkDir/Neuroergo_Hack/Predictions/{key}_{sub}.pickle\"\n",
    "    pickle.dump(clf, open(filename, 'wb'))\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {
    "jupyter": {
     "source_hidden": true
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "3\n",
      "model sknn_hard,Train time = 10.142722129821777, Predict time = 51.68991661071777, train acc = 1.0, test acc = 0.34675615212527966.\n",
      "[[0.06 0.94 0.  ]\n",
      " [0.06 0.94 0.  ]\n",
      " [0.04 0.92 0.04]]\n",
      "model NuSVC_hard,Train time = 10.911060094833374, Predict time = 51.96002817153931, train acc = 0.970917225950783, test acc = 0.34675615212527966.\n",
      "[[0.06 0.94 0.  ]\n",
      " [0.06 0.94 0.  ]\n",
      " [0.04 0.92 0.04]]\n",
      "model NuSVC_xgb_soft,Train time = 24.340091466903687, Predict time = 98.04278469085693, train acc = 0.9686800894854586, test acc = 0.7494407158836689.\n",
      "[[0.62 0.36 0.03]\n",
      " [0.19 0.68 0.13]\n",
      " [0.   0.05 0.95]]\n",
      "model xgb_SVC_weighted_hard,Train time = 20.40107274055481, Predict time = 89.57851123809814, train acc = 0.912751677852349, test acc = 0.5078299776286354.\n",
      "[[0.81 0.19 0.  ]\n",
      " [0.33 0.67 0.  ]\n",
      " [0.   0.96 0.04]]\n",
      "model SVC_hard,Train time = 11.442517757415771, Predict time = 52.30019736289978, train acc = 0.9261744966442953, test acc = 0.34675615212527966.\n",
      "[[0.06 0.94 0.  ]\n",
      " [0.06 0.94 0.  ]\n",
      " [0.04 0.92 0.04]]\n"
     ]
    }
   ],
   "source": [
    "sub = 3\n",
    "X_test,y_test = load_dat(sub,1)\n",
    "X_train,y_train = load_dat(sub,2)\n",
    "X_p = load_test_dat(sub,3)\n",
    "print(sub)\n",
    "for key,clf in array_clfs.items():\n",
    "    sub = 3\n",
    "    fname = f\"D:/Softwares/AnacondaWorkDir/Neuroergo_Hack/Predictions/{key}_{sub}.csv\"\n",
    "    \n",
    "    start = time.time()\n",
    "    clf.fit(X_train,y_train)\n",
    "    end = time.time()\n",
    "    t1 = end-start\n",
    "    \n",
    "    start = time.time()\n",
    "    p_train = clf.predict(X_train)\n",
    "    p_test = clf.predict(X_test)\n",
    "    end = time.time()\n",
    "    t2 = end-start\n",
    "    \n",
    "    c = np.around(cm(y_test,p_test,normalize='true'),2)\n",
    "    \n",
    "    p_pred = clf.predict(X_p)\n",
    "    pd.DataFrame(p_pred).to_csv(fname)\n",
    "    \n",
    "    print(f'model {key},Train time = {t1}, Predict time = {t2}, train acc = {accuracy_score(y_train,p_train)}, test acc = {accuracy_score(y_test,p_test)}.')\n",
    "    print(c)\n",
    "    \n",
    "    filename = f\"D:/Softwares/AnacondaWorkDir/Neuroergo_Hack/Predictions/{key}_{sub}.pickle\"\n",
    "    pickle.dump(clf, open(filename, 'wb'))\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {
    "jupyter": {
     "source_hidden": true
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "4\n",
      "model sknn_hard,Train time = 10.191020011901855, Predict time = 50.723453521728516, train acc = 1.0, test acc = 0.43847874720357943.\n",
      "[[0.74 0.19 0.07]\n",
      " [0.6  0.11 0.29]\n",
      " [0.48 0.05 0.46]]\n",
      "model NuSVC_hard,Train time = 10.701445817947388, Predict time = 50.81006979942322, train acc = 1.0, test acc = 0.44742729306487694.\n",
      "[[0.83 0.1  0.07]\n",
      " [0.64 0.02 0.34]\n",
      " [0.5  0.01 0.5 ]]\n",
      "model NuSVC_xgb_soft,Train time = 21.51299500465393, Predict time = 87.06529498100281, train acc = 1.0, test acc = 0.35570469798657717.\n",
      "[[0.03 0.7  0.28]\n",
      " [0.   0.23 0.77]\n",
      " [0.   0.19 0.81]]\n",
      "model xgb_SVC_weighted_hard,Train time = 21.812941551208496, Predict time = 86.57501101493835, train acc = 0.8411633109619687, test acc = 0.39373601789709173.\n",
      "[[0.03 0.9  0.07]\n",
      " [0.   0.66 0.34]\n",
      " [0.   0.5  0.5 ]]\n",
      "model SVC_hard,Train time = 10.832484245300293, Predict time = 51.037747621536255, train acc = 0.8680089485458613, test acc = 0.34675615212527966.\n",
      "[[0.19 0.73 0.07]\n",
      " [0.31 0.35 0.34]\n",
      " [0.33 0.17 0.5 ]]\n"
     ]
    }
   ],
   "source": [
    "sub = 4\n",
    "X_test,y_test = load_dat(sub,1)\n",
    "X_train,y_train = load_dat(sub,2)\n",
    "X_p = load_test_dat(sub,3)\n",
    "print(sub)\n",
    "for key,clf in array_clfs.items():\n",
    "    sub = 4\n",
    "    fname = f\"D:/Softwares/AnacondaWorkDir/Neuroergo_Hack/Predictions/{key}_{sub}.csv\"\n",
    "    \n",
    "    start = time.time()\n",
    "    clf.fit(X_train,y_train)\n",
    "    end = time.time()\n",
    "    t1 = end-start\n",
    "    \n",
    "    start = time.time()\n",
    "    p_train = clf.predict(X_train)\n",
    "    p_test = clf.predict(X_test)\n",
    "    end = time.time()\n",
    "    t2 = end-start\n",
    "    \n",
    "    c = np.around(cm(y_test,p_test,normalize='true'),2)\n",
    "    \n",
    "    p_pred = clf.predict(X_p)\n",
    "    pd.DataFrame(p_pred).to_csv(fname)\n",
    "    \n",
    "    print(f'model {key},Train time = {t1}, Predict time = {t2}, train acc = {accuracy_score(y_train,p_train)}, test acc = {accuracy_score(y_test,p_test)}.')\n",
    "    print(c)\n",
    "    \n",
    "    filename = f\"D:/Softwares/AnacondaWorkDir/Neuroergo_Hack/Predictions/{key}_{sub}.pickle\"\n",
    "    pickle.dump(clf, open(filename, 'wb'))\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {
    "jupyter": {
     "source_hidden": true
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "5\n",
      "model sknn_hard,Train time = 11.408822774887085, Predict time = 63.38685083389282, train acc = 1.0, test acc = 0.5682326621923938.\n",
      "[[0.81 0.03 0.16]\n",
      " [0.47 0.05 0.48]\n",
      " [0.15 0.01 0.84]]\n",
      "model NuSVC_hard,Train time = 12.177934646606445, Predict time = 63.07461452484131, train acc = 0.9776286353467561, test acc = 0.5592841163310962.\n",
      "[[0.81 0.03 0.17]\n",
      " [0.48 0.05 0.48]\n",
      " [0.17 0.01 0.83]]\n",
      "model NuSVC_xgb_soft,Train time = 25.05189800262451, Predict time = 112.55902242660522, train acc = 0.9776286353467561, test acc = 0.6442953020134228.\n",
      "[[0.8  0.11 0.09]\n",
      " [0.39 0.35 0.26]\n",
      " [0.15 0.06 0.79]]\n",
      "model xgb_SVC_weighted_hard,Train time = 24.326865911483765, Predict time = 114.79849982261658, train acc = 0.6442953020134228, test acc = 0.5883668903803132.\n",
      "[[0.83 0.05 0.12]\n",
      " [0.5  0.21 0.29]\n",
      " [0.21 0.07 0.72]]\n",
      "model SVC_hard,Train time = 12.252467393875122, Predict time = 64.82497787475586, train acc = 0.7069351230425056, test acc = 0.5682326621923938.\n",
      "[[0.8  0.03 0.17]\n",
      " [0.44 0.09 0.48]\n",
      " [0.16 0.02 0.82]]\n"
     ]
    }
   ],
   "source": [
    "sub = 5\n",
    "X_test,y_test = load_dat(sub,1)\n",
    "X_train,y_train = load_dat(sub,2)\n",
    "X_p = load_test_dat(sub,3)\n",
    "print(sub)\n",
    "for key,clf in array_clfs.items():\n",
    "    sub = 5\n",
    "    fname = f\"D:/Softwares/AnacondaWorkDir/Neuroergo_Hack/Predictions/{key}_{sub}.csv\"\n",
    "    \n",
    "    start = time.time()\n",
    "    clf.fit(X_train,y_train)\n",
    "    end = time.time()\n",
    "    t1 = end-start\n",
    "    \n",
    "    start = time.time()\n",
    "    p_train = clf.predict(X_train)\n",
    "    p_test = clf.predict(X_test)\n",
    "    end = time.time()\n",
    "    t2 = end-start\n",
    "    \n",
    "    c = np.around(cm(y_test,p_test,normalize='true'),2)\n",
    "    \n",
    "    p_pred = clf.predict(X_p)\n",
    "    pd.DataFrame(p_pred).to_csv(fname)\n",
    "    \n",
    "    print(f'model {key},Train time = {t1}, Predict time = {t2}, train acc = {accuracy_score(y_train,p_train)}, test acc = {accuracy_score(y_test,p_test)}.')\n",
    "    print(c)\n",
    "    \n",
    "    filename = f\"D:/Softwares/AnacondaWorkDir/Neuroergo_Hack/Predictions/{key}_{sub}.pickle\"\n",
    "    pickle.dump(clf, open(filename, 'wb'))\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {
    "jupyter": {
     "source_hidden": true
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "6\n",
      "model sknn_hard,Train time = 10.731504917144775, Predict time = 53.75531220436096, train acc = 1.0, test acc = 0.6062639821029083.\n",
      "[[0.81 0.03 0.16]\n",
      " [0.22 0.21 0.57]\n",
      " [0.13 0.07 0.81]]\n",
      "model NuSVC_hard,Train time = 11.640982866287231, Predict time = 53.52612376213074, train acc = 0.9798657718120806, test acc = 0.6062639821029083.\n",
      "[[0.85 0.   0.15]\n",
      " [0.23 0.14 0.63]\n",
      " [0.14 0.03 0.83]]\n",
      "model NuSVC_xgb_soft,Train time = 20.55870270729065, Predict time = 92.89141273498535, train acc = 0.9798657718120806, test acc = 0.5883668903803132.\n",
      "[[0.82 0.03 0.15]\n",
      " [0.16 0.16 0.68]\n",
      " [0.11 0.1  0.79]]\n",
      "model xgb_SVC_weighted_hard,Train time = 22.350396871566772, Predict time = 93.02756381034851, train acc = 0.8411633109619687, test acc = 0.5883668903803132.\n",
      "[[0.78 0.05 0.17]\n",
      " [0.16 0.29 0.55]\n",
      " [0.15 0.15 0.7 ]]\n",
      "model SVC_hard,Train time = 11.53057312965393, Predict time = 53.90191149711609, train acc = 0.9172259507829977, test acc = 0.5816554809843401.\n",
      "[[0.83 0.01 0.16]\n",
      " [0.26 0.25 0.5 ]\n",
      " [0.25 0.08 0.67]]\n"
     ]
    }
   ],
   "source": [
    "sub = 6\n",
    "X_test,y_test = load_dat(sub,1)\n",
    "X_train,y_train = load_dat(sub,2)\n",
    "X_p = load_test_dat(sub,3)\n",
    "print(sub)\n",
    "for key,clf in array_clfs.items():\n",
    "    sub = 6\n",
    "    fname = f\"D:/Softwares/AnacondaWorkDir/Neuroergo_Hack/Predictions/{key}_{sub}.csv\"\n",
    "    \n",
    "    start = time.time()\n",
    "    clf.fit(X_train,y_train)\n",
    "    end = time.time()\n",
    "    t1 = end-start\n",
    "    \n",
    "    start = time.time()\n",
    "    p_train = clf.predict(X_train)\n",
    "    p_test = clf.predict(X_test)\n",
    "    end = time.time()\n",
    "    t2 = end-start\n",
    "    \n",
    "    c = np.around(cm(y_test,p_test,normalize='true'),2)\n",
    "    \n",
    "    p_pred = clf.predict(X_p)\n",
    "    pd.DataFrame(p_pred).to_csv(fname)\n",
    "    \n",
    "    print(f'model {key},Train time = {t1}, Predict time = {t2}, train acc = {accuracy_score(y_train,p_train)}, test acc = {accuracy_score(y_test,p_test)}.')\n",
    "    print(c)\n",
    "    \n",
    "    filename = f\"D:/Softwares/AnacondaWorkDir/Neuroergo_Hack/Predictions/{key}_{sub}.pickle\"\n",
    "    pickle.dump(clf, open(filename, 'wb'))\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {
    "jupyter": {
     "source_hidden": true
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "7\n",
      "model sknn_hard,Train time = 10.101146697998047, Predict time = 53.06065011024475, train acc = 1.0, test acc = 0.4541387024608501.\n",
      "[[0.92 0.01 0.07]\n",
      " [0.62 0.11 0.26]\n",
      " [0.58 0.09 0.33]]\n",
      "model NuSVC_hard,Train time = 10.892117738723755, Predict time = 53.61599636077881, train acc = 0.9865771812080537, test acc = 0.42058165548098436.\n",
      "[[0.91 0.01 0.07]\n",
      " [0.73 0.06 0.21]\n",
      " [0.62 0.09 0.29]]\n",
      "model NuSVC_xgb_soft,Train time = 23.075266361236572, Predict time = 92.45352077484131, train acc = 0.9821029082774049, test acc = 0.47874720357941836.\n",
      "[[0.81 0.1  0.09]\n",
      " [0.31 0.17 0.52]\n",
      " [0.27 0.28 0.45]]\n",
      "model xgb_SVC_weighted_hard,Train time = 23.102816104888916, Predict time = 92.66772651672363, train acc = 0.6331096196868009, test acc = 0.49217002237136465.\n",
      "[[0.9  0.03 0.07]\n",
      " [0.48 0.25 0.28]\n",
      " [0.45 0.22 0.33]]\n",
      "model SVC_hard,Train time = 10.943039417266846, Predict time = 54.119901180267334, train acc = 0.6778523489932886, test acc = 0.4988814317673378.\n",
      "[[0.92 0.01 0.07]\n",
      " [0.49 0.25 0.26]\n",
      " [0.46 0.21 0.33]]\n"
     ]
    }
   ],
   "source": [
    "sub = 7\n",
    "X_test,y_test = load_dat(sub,1)\n",
    "X_train,y_train = load_dat(sub,2)\n",
    "X_p = load_test_dat(sub,3)\n",
    "print(sub)\n",
    "for key,clf in array_clfs.items():\n",
    "    sub = 7\n",
    "    fname = f\"D:/Softwares/AnacondaWorkDir/Neuroergo_Hack/Predictions/{key}_{sub}.csv\"\n",
    "    \n",
    "    start = time.time()\n",
    "    clf.fit(X_train,y_train)\n",
    "    end = time.time()\n",
    "    t1 = end-start\n",
    "    \n",
    "    start = time.time()\n",
    "    p_train = clf.predict(X_train)\n",
    "    p_test = clf.predict(X_test)\n",
    "    end = time.time()\n",
    "    t2 = end-start\n",
    "    \n",
    "    c = np.around(cm(y_test,p_test,normalize='true'),2)\n",
    "    \n",
    "    p_pred = clf.predict(X_p)\n",
    "    pd.DataFrame(p_pred).to_csv(fname)\n",
    "    \n",
    "    print(f'model {key},Train time = {t1}, Predict time = {t2}, train acc = {accuracy_score(y_train,p_train)}, test acc = {accuracy_score(y_test,p_test)}.')\n",
    "    print(c)\n",
    "    \n",
    "    filename = f\"D:/Softwares/AnacondaWorkDir/Neuroergo_Hack/Predictions/{key}_{sub}.pickle\"\n",
    "    pickle.dump(clf, open(filename, 'wb'))\n",
    "  "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {
    "jupyter": {
     "source_hidden": true
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "8\n",
      "model sknn_hard,Train time = 10.6277174949646, Predict time = 53.508241176605225, train acc = 1.0, test acc = 0.5883668903803132.\n",
      "[[0.89 0.07 0.04]\n",
      " [0.47 0.21 0.32]\n",
      " [0.3  0.04 0.66]]\n",
      "model NuSVC_hard,Train time = 12.01052975654602, Predict time = 53.799922704696655, train acc = 0.9977628635346756, test acc = 0.5995525727069351.\n",
      "[[0.9  0.09 0.01]\n",
      " [0.42 0.31 0.27]\n",
      " [0.33 0.08 0.59]]\n",
      "model NuSVC_xgb_soft,Train time = 23.59673833847046, Predict time = 92.65569281578064, train acc = 0.9977628635346756, test acc = 0.6733780760626398.\n",
      "[[0.8  0.19 0.01]\n",
      " [0.03 0.49 0.48]\n",
      " [0.07 0.19 0.73]]\n",
      "model xgb_SVC_weighted_hard,Train time = 23.834446907043457, Predict time = 92.56416606903076, train acc = 0.6890380313199105, test acc = 0.6577181208053692.\n",
      "[[0.81 0.09 0.1 ]\n",
      " [0.07 0.39 0.54]\n",
      " [0.1  0.12 0.78]]\n",
      "model SVC_hard,Train time = 11.6311354637146, Predict time = 53.72228002548218, train acc = 0.7606263982102909, test acc = 0.6286353467561522.\n",
      "[[0.84 0.07 0.09]\n",
      " [0.21 0.25 0.54]\n",
      " [0.14 0.06 0.8 ]]\n"
     ]
    }
   ],
   "source": [
    "sub = 8\n",
    "X_test,y_test = load_dat(sub,1)\n",
    "X_train,y_train = load_dat(sub,2)\n",
    "X_p = load_test_dat(sub,3)\n",
    "print(sub)\n",
    "for key,clf in array_clfs.items():\n",
    "    sub = 8\n",
    "    fname = f\"D:/Softwares/AnacondaWorkDir/Neuroergo_Hack/Predictions/{key}_{sub}.csv\"\n",
    "    \n",
    "    start = time.time()\n",
    "    clf.fit(X_train,y_train)\n",
    "    end = time.time()\n",
    "    t1 = end-start\n",
    "    \n",
    "    start = time.time()\n",
    "    p_train = clf.predict(X_train)\n",
    "    p_test = clf.predict(X_test)\n",
    "    end = time.time()\n",
    "    t2 = end-start\n",
    "    \n",
    "    c = np.around(cm(y_test,p_test,normalize='true'),2)\n",
    "    \n",
    "    p_pred = clf.predict(X_p)\n",
    "    pd.DataFrame(p_pred).to_csv(fname)\n",
    "    \n",
    "    print(f'model {key},Train time = {t1}, Predict time = {t2}, train acc = {accuracy_score(y_train,p_train)}, test acc = {accuracy_score(y_test,p_test)}.')\n",
    "    print(c)\n",
    "    \n",
    "    filename = f\"D:/Softwares/AnacondaWorkDir/Neuroergo_Hack/Predictions/{key}_{sub}.pickle\"\n",
    "    pickle.dump(clf, open(filename, 'wb'))\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {
    "jupyter": {
     "source_hidden": true
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "9\n",
      "model sknn_hard,Train time = 11.151219367980957, Predict time = 58.0589017868042, train acc = 1.0, test acc = 0.4161073825503356.\n",
      "[[0.32 0.68 0.  ]\n",
      " [0.09 0.91 0.  ]\n",
      " [0.01 0.97 0.02]]\n",
      "model NuSVC_hard,Train time = 11.570015668869019, Predict time = 57.99423003196716, train acc = 0.9530201342281879, test acc = 0.4161073825503356.\n",
      "[[0.32 0.68 0.  ]\n",
      " [0.09 0.91 0.  ]\n",
      " [0.01 0.97 0.02]]\n",
      "model NuSVC_xgb_soft,Train time = 20.15811514854431, Predict time = 102.14719414710999, train acc = 0.9507829977628636, test acc = 0.5637583892617449.\n",
      "[[0.6  0.4  0.01]\n",
      " [0.28 0.6  0.11]\n",
      " [0.02 0.49 0.49]]\n",
      "model xgb_SVC_weighted_hard,Train time = 21.491820573806763, Predict time = 103.35075497627258, train acc = 0.8456375838926175, test acc = 0.4541387024608501.\n",
      "[[0.68 0.32 0.  ]\n",
      " [0.32 0.68 0.  ]\n",
      " [0.02 0.98 0.  ]]\n",
      "model SVC_hard,Train time = 11.754506587982178, Predict time = 58.101601362228394, train acc = 0.9485458612975392, test acc = 0.4161073825503356.\n",
      "[[0.32 0.68 0.  ]\n",
      " [0.09 0.91 0.  ]\n",
      " [0.01 0.97 0.02]]\n"
     ]
    }
   ],
   "source": [
    "sub = 9\n",
    "X_test,y_test = load_dat(sub,1)\n",
    "X_train,y_train = load_dat(sub,2)\n",
    "X_p = load_test_dat(sub,3)\n",
    "print(sub)\n",
    "for key,clf in array_clfs.items():\n",
    "    sub = 9\n",
    "    fname = f\"D:/Softwares/AnacondaWorkDir/Neuroergo_Hack/Predictions/{key}_{sub}.csv\"\n",
    "    \n",
    "    start = time.time()\n",
    "    clf.fit(X_train,y_train)\n",
    "    end = time.time()\n",
    "    t1 = end-start\n",
    "    \n",
    "    start = time.time()\n",
    "    p_train = clf.predict(X_train)\n",
    "    p_test = clf.predict(X_test)\n",
    "    end = time.time()\n",
    "    t2 = end-start\n",
    "    \n",
    "    c = np.around(cm(y_test,p_test,normalize='true'),2)\n",
    "    \n",
    "    p_pred = clf.predict(X_p)\n",
    "    pd.DataFrame(p_pred).to_csv(fname)\n",
    "    \n",
    "    print(f'model {key},Train time = {t1}, Predict time = {t2}, train acc = {accuracy_score(y_train,p_train)}, test acc = {accuracy_score(y_test,p_test)}.')\n",
    "    print(c)\n",
    "    \n",
    "    filename = f\"D:/Softwares/AnacondaWorkDir/Neuroergo_Hack/Predictions/{key}_{sub}.pickle\"\n",
    "    pickle.dump(clf, open(filename, 'wb'))\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {
    "jupyter": {
     "source_hidden": true
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "10\n",
      "model sknn_hard,Train time = 9.099262475967407, Predict time = 42.58891725540161, train acc = 1.0, test acc = 0.5123042505592841.\n",
      "[[0.   1.   0.  ]\n",
      " [0.   1.   0.  ]\n",
      " [0.01 0.46 0.54]]\n",
      "model NuSVC_hard,Train time = 10.042159795761108, Predict time = 42.3510377407074, train acc = 0.9977628635346756, test acc = 0.5190156599552572.\n",
      "[[0.   1.   0.  ]\n",
      " [0.   1.   0.  ]\n",
      " [0.   0.44 0.56]]\n",
      "model NuSVC_xgb_soft,Train time = 18.051048517227173, Predict time = 70.19773292541504, train acc = 1.0, test acc = 0.5123042505592841.\n",
      "[[0.02 0.98 0.  ]\n",
      " [0.03 0.97 0.  ]\n",
      " [0.03 0.43 0.54]]\n",
      "model xgb_SVC_weighted_hard,Train time = 17.79991126060486, Predict time = 70.0085277557373, train acc = 0.9753914988814317, test acc = 0.43847874720357943.\n",
      "[[0.02 0.98 0.  ]\n",
      " [0.02 0.98 0.  ]\n",
      " [0.   0.68 0.32]]\n",
      "model SVC_hard,Train time = 9.745293378829956, Predict time = 42.45206689834595, train acc = 0.9776286353467561, test acc = 0.5190156599552572.\n",
      "[[0.   1.   0.  ]\n",
      " [0.   1.   0.  ]\n",
      " [0.01 0.44 0.56]]\n"
     ]
    }
   ],
   "source": [
    "sub = 10\n",
    "X_test,y_test = load_dat(sub,1)\n",
    "X_train,y_train = load_dat(sub,2)\n",
    "X_p = load_test_dat(sub,3)\n",
    "print(sub)\n",
    "for key,clf in array_clfs.items():\n",
    "    sub = 10\n",
    "    fname = f\"D:/Softwares/AnacondaWorkDir/Neuroergo_Hack/Predictions/{key}_{sub}.csv\"\n",
    "    \n",
    "    start = time.time()\n",
    "    clf.fit(X_train,y_train)\n",
    "    end = time.time()\n",
    "    t1 = end-start\n",
    "    \n",
    "    start = time.time()\n",
    "    p_train = clf.predict(X_train)\n",
    "    p_test = clf.predict(X_test)\n",
    "    end = time.time()\n",
    "    t2 = end-start\n",
    "    \n",
    "    c = np.around(cm(y_test,p_test,normalize='true'),2)\n",
    "    \n",
    "    p_pred = clf.predict(X_p)\n",
    "    pd.DataFrame(p_pred).to_csv(fname)\n",
    "    \n",
    "    print(f'model {key},Train time = {t1}, Predict time = {t2}, train acc = {accuracy_score(y_train,p_train)}, test acc = {accuracy_score(y_test,p_test)}.')\n",
    "    print(c)\n",
    "    \n",
    "    filename = f\"D:/Softwares/AnacondaWorkDir/Neuroergo_Hack/Predictions/{key}_{sub}.pickle\"\n",
    "    pickle.dump(clf, open(filename, 'wb'))\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {
    "jupyter": {
     "source_hidden": true
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "11\n",
      "model sknn_hard,Train time = 10.392640590667725, Predict time = 56.61525797843933, train acc = 1.0, test acc = 0.3333333333333333.\n",
      "[[1. 0. 0.]\n",
      " [1. 0. 0.]\n",
      " [1. 0. 0.]]\n",
      "model NuSVC_hard,Train time = 11.21146821975708, Predict time = 56.46492862701416, train acc = 0.9463087248322147, test acc = 0.33557046979865773.\n",
      "[[0.99 0.01 0.  ]\n",
      " [0.99 0.01 0.  ]\n",
      " [0.98 0.02 0.  ]]\n",
      "model NuSVC_xgb_soft,Train time = 21.515706539154053, Predict time = 98.25250244140625, train acc = 0.9463087248322147, test acc = 0.5346756152125279.\n",
      "[[0.77 0.23 0.01]\n",
      " [0.16 0.79 0.05]\n",
      " [0.05 0.9  0.05]]\n",
      "model xgb_SVC_weighted_hard,Train time = 21.182105779647827, Predict time = 98.26710748672485, train acc = 0.8590604026845637, test acc = 0.5324384787472036.\n",
      "[[0.77 0.23 0.  ]\n",
      " [0.17 0.83 0.  ]\n",
      " [0.05 0.95 0.  ]]\n",
      "model SVC_hard,Train time = 11.400665998458862, Predict time = 56.4250807762146, train acc = 0.9284116331096197, test acc = 0.3333333333333333.\n",
      "[[1. 0. 0.]\n",
      " [1. 0. 0.]\n",
      " [1. 0. 0.]]\n"
     ]
    }
   ],
   "source": [
    "sub = 11\n",
    "X_test,y_test = load_dat(sub,1)\n",
    "X_train,y_train = load_dat(sub,2)\n",
    "X_p = load_test_dat(sub,3)\n",
    "print(sub)\n",
    "for key,clf in array_clfs.items():\n",
    "    sub = 11\n",
    "    fname = f\"D:/Softwares/AnacondaWorkDir/Neuroergo_Hack/Predictions/{key}_{sub}.csv\"\n",
    "    \n",
    "    start = time.time()\n",
    "    clf.fit(X_train,y_train)\n",
    "    end = time.time()\n",
    "    t1 = end-start\n",
    "    \n",
    "    start = time.time()\n",
    "    p_train = clf.predict(X_train)\n",
    "    p_test = clf.predict(X_test)\n",
    "    end = time.time()\n",
    "    t2 = end-start\n",
    "    \n",
    "    c = np.around(cm(y_test,p_test,normalize='true'),2)\n",
    "    \n",
    "    p_pred = clf.predict(X_p)\n",
    "    pd.DataFrame(p_pred).to_csv(fname)\n",
    "    \n",
    "    print(f'model {key},Train time = {t1}, Predict time = {t2}, train acc = {accuracy_score(y_train,p_train)}, test acc = {accuracy_score(y_test,p_test)}.')\n",
    "    print(c)\n",
    "    \n",
    "    filename = f\"D:/Softwares/AnacondaWorkDir/Neuroergo_Hack/Predictions/{key}_{sub}.pickle\"\n",
    "    pickle.dump(clf, open(filename, 'wb'))\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {
    "jupyter": {
     "source_hidden": true
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "12\n",
      "model sknn_hard,Train time = 10.635932207107544, Predict time = 53.745107889175415, train acc = 1.0, test acc = 0.43400447427293065.\n",
      "[[0.53 0.34 0.13]\n",
      " [0.4  0.52 0.07]\n",
      " [0.19 0.56 0.25]]\n",
      "model NuSVC_hard,Train time = 11.382272481918335, Predict time = 53.92194676399231, train acc = 0.9955257270693513, test acc = 0.5167785234899329.\n",
      "[[0.5  0.17 0.33]\n",
      " [0.3  0.58 0.11]\n",
      " [0.11 0.42 0.47]]\n",
      "model NuSVC_xgb_soft,Train time = 28.693744897842407, Predict time = 92.57937145233154, train acc = 0.9977628635346756, test acc = 0.4966442953020134.\n",
      "[[0.51 0.34 0.15]\n",
      " [0.34 0.57 0.09]\n",
      " [0.13 0.46 0.41]]\n",
      "model xgb_SVC_weighted_hard,Train time = 28.42510986328125, Predict time = 93.70217537879944, train acc = 0.6129753914988815, test acc = 0.4519015659955257.\n",
      "[[0.46 0.52 0.03]\n",
      " [0.26 0.72 0.03]\n",
      " [0.23 0.59 0.18]]\n",
      "model SVC_hard,Train time = 11.379311323165894, Predict time = 54.249030351638794, train acc = 0.6442953020134228, test acc = 0.5279642058165548.\n",
      "[[0.42 0.44 0.14]\n",
      " [0.22 0.66 0.11]\n",
      " [0.19 0.3  0.5 ]]\n"
     ]
    }
   ],
   "source": [
    "sub = 12\n",
    "X_test,y_test = load_dat(sub,1)\n",
    "X_train,y_train = load_dat(sub,2)\n",
    "X_p = load_test_dat(sub,3)\n",
    "print(sub)\n",
    "for key,clf in array_clfs.items():\n",
    "    sub = 12\n",
    "    fname = f\"D:/Softwares/AnacondaWorkDir/Neuroergo_Hack/Predictions/{key}_{sub}.csv\"\n",
    "    \n",
    "    start = time.time()\n",
    "    clf.fit(X_train,y_train)\n",
    "    end = time.time()\n",
    "    t1 = end-start\n",
    "    \n",
    "    start = time.time()\n",
    "    p_train = clf.predict(X_train)\n",
    "    p_test = clf.predict(X_test)\n",
    "    end = time.time()\n",
    "    t2 = end-start\n",
    "    \n",
    "    c = np.around(cm(y_test,p_test,normalize='true'),2)\n",
    "    \n",
    "    p_pred = clf.predict(X_p)\n",
    "    pd.DataFrame(p_pred).to_csv(fname)\n",
    "    \n",
    "    print(f'model {key},Train time = {t1}, Predict time = {t2}, train acc = {accuracy_score(y_train,p_train)}, test acc = {accuracy_score(y_test,p_test)}.')\n",
    "    print(c)\n",
    "    \n",
    "    filename = f\"D:/Softwares/AnacondaWorkDir/Neuroergo_Hack/Predictions/{key}_{sub}.pickle\"\n",
    "    pickle.dump(clf, open(filename, 'wb'))\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {
    "jupyter": {
     "source_hidden": true
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "13\n",
      "model sknn_hard,Train time = 11.50284743309021, Predict time = 52.046220541000366, train acc = 1.0, test acc = 0.33109619686800895.\n",
      "[[0.46 0.04 0.5 ]\n",
      " [0.21 0.07 0.72]\n",
      " [0.34 0.2  0.46]]\n",
      "model NuSVC_hard,Train time = 11.380378007888794, Predict time = 52.36738085746765, train acc = 0.9932885906040269, test acc = 0.37360178970917224.\n",
      "[[0.67 0.05 0.28]\n",
      " [0.36 0.09 0.55]\n",
      " [0.4  0.24 0.36]]\n",
      "model NuSVC_xgb_soft,Train time = 23.037548303604126, Predict time = 89.27395296096802, train acc = 0.9932885906040269, test acc = 0.4228187919463087.\n",
      "[[0.87 0.07 0.07]\n",
      " [0.49 0.23 0.28]\n",
      " [0.46 0.37 0.17]]\n",
      "model xgb_SVC_weighted_hard,Train time = 22.91073226928711, Predict time = 89.50749802589417, train acc = 0.6823266219239373, test acc = 0.36017897091722595.\n",
      "[[0.67 0.06 0.27]\n",
      " [0.45 0.13 0.42]\n",
      " [0.47 0.25 0.28]]\n",
      "model SVC_hard,Train time = 11.536175966262817, Predict time = 52.183273792266846, train acc = 0.7740492170022372, test acc = 0.3400447427293065.\n",
      "[[0.54 0.05 0.4 ]\n",
      " [0.25 0.09 0.66]\n",
      " [0.37 0.24 0.39]]\n"
     ]
    }
   ],
   "source": [
    "sub = 13\n",
    "X_test,y_test = load_dat(sub,1)\n",
    "X_train,y_train = load_dat(sub,2)\n",
    "X_p = load_test_dat(sub,3)\n",
    "print(sub)\n",
    "for key,clf in array_clfs.items():\n",
    "    sub = 13\n",
    "    fname = f\"D:/Softwares/AnacondaWorkDir/Neuroergo_Hack/Predictions/{key}_{sub}.csv\"\n",
    "    \n",
    "    start = time.time()\n",
    "    clf.fit(X_train,y_train)\n",
    "    end = time.time()\n",
    "    t1 = end-start\n",
    "    \n",
    "    start = time.time()\n",
    "    p_train = clf.predict(X_train)\n",
    "    p_test = clf.predict(X_test)\n",
    "    end = time.time()\n",
    "    t2 = end-start\n",
    "    \n",
    "    c = np.around(cm(y_test,p_test,normalize='true'),2)\n",
    "    \n",
    "    p_pred = clf.predict(X_p)\n",
    "    pd.DataFrame(p_pred).to_csv(fname)\n",
    "    \n",
    "    print(f'model {key},Train time = {t1}, Predict time = {t2}, train acc = {accuracy_score(y_train,p_train)}, test acc = {accuracy_score(y_test,p_test)}.')\n",
    "    print(c)\n",
    "    \n",
    "    filename = f\"D:/Softwares/AnacondaWorkDir/Neuroergo_Hack/Predictions/{key}_{sub}.pickle\"\n",
    "    pickle.dump(clf, open(filename, 'wb'))\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "metadata": {
    "jupyter": {
     "source_hidden": true
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "14\n",
      "model sknn_hard,Train time = 10.733843564987183, Predict time = 56.404130697250366, train acc = 1.0, test acc = 0.4720357941834452.\n",
      "[[0.58 0.04 0.38]\n",
      " [0.34 0.01 0.65]\n",
      " [0.17 0.   0.83]]\n",
      "model NuSVC_hard,Train time = 11.94075083732605, Predict time = 57.117955684661865, train acc = 1.0, test acc = 0.44742729306487694.\n",
      "[[0.64 0.05 0.31]\n",
      " [0.41 0.01 0.58]\n",
      " [0.31 0.   0.69]]\n",
      "model NuSVC_xgb_soft,Train time = 20.419580459594727, Predict time = 98.82681083679199, train acc = 1.0, test acc = 0.37360178970917224.\n",
      "[[0.6  0.35 0.05]\n",
      " [0.57 0.03 0.4 ]\n",
      " [0.4  0.12 0.48]]\n",
      "model xgb_SVC_weighted_hard,Train time = 21.002515077590942, Predict time = 98.75317192077637, train acc = 0.8031319910514542, test acc = 0.43400447427293065.\n",
      "[[0.72 0.19 0.09]\n",
      " [0.52 0.03 0.46]\n",
      " [0.42 0.03 0.56]]\n",
      "model SVC_hard,Train time = 11.757803440093994, Predict time = 56.85169863700867, train acc = 0.8970917225950783, test acc = 0.4272930648769575.\n",
      "[[0.65 0.04 0.31]\n",
      " [0.42 0.01 0.58]\n",
      " [0.38 0.   0.62]]\n"
     ]
    }
   ],
   "source": [
    "sub = 14\n",
    "X_test,y_test = load_dat(sub,1)\n",
    "X_train,y_train = load_dat(sub,2)\n",
    "X_p = load_test_dat(sub,3)\n",
    "print(sub)\n",
    "for key,clf in array_clfs.items():\n",
    "    sub = 14\n",
    "    fname = f\"D:/Softwares/AnacondaWorkDir/Neuroergo_Hack/Predictions/{key}_{sub}.csv\"\n",
    "    \n",
    "    start = time.time()\n",
    "    clf.fit(X_train,y_train)\n",
    "    end = time.time()\n",
    "    t1 = end-start\n",
    "    \n",
    "    start = time.time()\n",
    "    p_train = clf.predict(X_train)\n",
    "    p_test = clf.predict(X_test)\n",
    "    end = time.time()\n",
    "    t2 = end-start\n",
    "    \n",
    "    c = np.around(cm(y_test,p_test,normalize='true'),2)\n",
    "    \n",
    "    p_pred = clf.predict(X_p)\n",
    "    pd.DataFrame(p_pred).to_csv(fname)\n",
    "    \n",
    "    print(f'model {key},Train time = {t1}, Predict time = {t2}, train acc = {accuracy_score(y_train,p_train)}, test acc = {accuracy_score(y_test,p_test)}.')\n",
    "    print(c)\n",
    "    \n",
    "    filename = f\"D:/Softwares/AnacondaWorkDir/Neuroergo_Hack/Predictions/{key}_{sub}.pickle\"\n",
    "    pickle.dump(clf, open(filename, 'wb'))\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "metadata": {
    "jupyter": {
     "source_hidden": true
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "15\n",
      "model sknn_hard,Train time = 10.140229225158691, Predict time = 48.88445591926575, train acc = 1.0, test acc = 0.4854586129753915.\n",
      "[[0.73 0.25 0.02]\n",
      " [0.35 0.23 0.42]\n",
      " [0.36 0.14 0.5 ]]\n",
      "model NuSVC_hard,Train time = 10.730815887451172, Predict time = 49.31906270980835, train acc = 1.0, test acc = 0.5055928411633109.\n",
      "[[0.81 0.17 0.01]\n",
      " [0.4  0.2  0.4 ]\n",
      " [0.35 0.15 0.5 ]]\n",
      "model NuSVC_xgb_soft,Train time = 23.741156816482544, Predict time = 83.92739486694336, train acc = 0.9977628635346756, test acc = 0.44966442953020136.\n",
      "[[0.67 0.32 0.01]\n",
      " [0.38 0.26 0.37]\n",
      " [0.34 0.24 0.42]]\n",
      "model xgb_SVC_weighted_hard,Train time = 23.543327569961548, Predict time = 84.55523133277893, train acc = 0.7360178970917226, test acc = 0.4161073825503356.\n",
      "[[0.6  0.39 0.01]\n",
      " [0.39 0.36 0.26]\n",
      " [0.37 0.34 0.29]]\n",
      "model SVC_hard,Train time = 10.82939338684082, Predict time = 49.89279389381409, train acc = 0.8165548098434005, test acc = 0.4272930648769575.\n",
      "[[0.6  0.39 0.01]\n",
      " [0.39 0.33 0.28]\n",
      " [0.37 0.28 0.36]]\n"
     ]
    }
   ],
   "source": [
    "sub = 15\n",
    "X_test,y_test = load_dat(sub,1)\n",
    "X_train,y_train = load_dat(sub,2)\n",
    "X_p = load_test_dat(sub,3)\n",
    "print(sub)\n",
    "for key,clf in array_clfs.items():\n",
    "    sub = 15\n",
    "    fname = f\"D:/Softwares/AnacondaWorkDir/Neuroergo_Hack/Predictions/{key}_{sub}.csv\"\n",
    "    \n",
    "    start = time.time()\n",
    "    clf.fit(X_train,y_train)\n",
    "    end = time.time()\n",
    "    t1 = end-start\n",
    "    \n",
    "    start = time.time()\n",
    "    p_train = clf.predict(X_train)\n",
    "    p_test = clf.predict(X_test)\n",
    "    end = time.time()\n",
    "    t2 = end-start\n",
    "    \n",
    "    c = np.around(cm(y_test,p_test,normalize='true'),2)\n",
    "    \n",
    "    p_pred = clf.predict(X_p)\n",
    "    pd.DataFrame(p_pred).to_csv(fname)\n",
    "    \n",
    "    print(f'model {key},Train time = {t1}, Predict time = {t2}, train acc = {accuracy_score(y_train,p_train)}, test acc = {accuracy_score(y_test,p_test)}.')\n",
    "    print(c)\n",
    "    \n",
    "    filename = f\"D:/Softwares/AnacondaWorkDir/Neuroergo_Hack/Predictions/{key}_{sub}.pickle\"\n",
    "    pickle.dump(clf, open(filename, 'wb'))\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "collapsed": true,
    "jupyter": {
     "outputs_hidden": true,
     "source_hidden": true
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "1\n"
     ]
    },
    {
     "ename": "AttributeError",
     "evalue": "predict_proba is not available when voting='hard'",
     "output_type": "error",
     "traceback": [
      "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[1;31mAttributeError\u001b[0m                            Traceback (most recent call last)",
      "\u001b[1;32m<ipython-input-8-e0a676bb17f5>\u001b[0m in \u001b[0;36m<module>\u001b[1;34m\u001b[0m\n\u001b[0;32m     17\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     18\u001b[0m     \u001b[0mstart\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mtime\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mtime\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m---> 19\u001b[1;33m     \u001b[0mp_train\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0melvl2_s\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mpredict\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mX_train\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m     20\u001b[0m     \u001b[0mp_test\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0melvl2_s\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mpredict\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mX_test\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     21\u001b[0m     \u001b[0mend\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mtime\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mtime\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mD:\\Softwares\\Anaconda3\\envs\\neuroerg\\lib\\site-packages\\sklearn\\ensemble\\_voting.py\u001b[0m in \u001b[0;36mpredict\u001b[1;34m(self, X)\u001b[0m\n\u001b[0;32m    307\u001b[0m         \u001b[0mcheck_is_fitted\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    308\u001b[0m         \u001b[1;32mif\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mvoting\u001b[0m \u001b[1;33m==\u001b[0m \u001b[1;34m'soft'\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 309\u001b[1;33m             \u001b[0mmaj\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mnp\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0margmax\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mpredict_proba\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mX\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0maxis\u001b[0m\u001b[1;33m=\u001b[0m\u001b[1;36m1\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    310\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    311\u001b[0m         \u001b[1;32melse\u001b[0m\u001b[1;33m:\u001b[0m  \u001b[1;31m# 'hard' voting\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mD:\\Softwares\\Anaconda3\\envs\\neuroerg\\lib\\site-packages\\sklearn\\ensemble\\_voting.py\u001b[0m in \u001b[0;36m_predict_proba\u001b[1;34m(self, X)\u001b[0m\n\u001b[0;32m    327\u001b[0m         \u001b[1;34m\"\"\"Predict class probabilities for X in 'soft' voting.\"\"\"\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    328\u001b[0m         \u001b[0mcheck_is_fitted\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 329\u001b[1;33m         avg = np.average(self._collect_probas(X), axis=0,\n\u001b[0m\u001b[0;32m    330\u001b[0m                          weights=self._weights_not_none)\n\u001b[0;32m    331\u001b[0m         \u001b[1;32mreturn\u001b[0m \u001b[0mavg\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mD:\\Softwares\\Anaconda3\\envs\\neuroerg\\lib\\site-packages\\sklearn\\ensemble\\_voting.py\u001b[0m in \u001b[0;36m_collect_probas\u001b[1;34m(self, X)\u001b[0m\n\u001b[0;32m    322\u001b[0m     \u001b[1;32mdef\u001b[0m \u001b[0m_collect_probas\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mX\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    323\u001b[0m         \u001b[1;34m\"\"\"Collect results from clf.predict calls.\"\"\"\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 324\u001b[1;33m         \u001b[1;32mreturn\u001b[0m \u001b[0mnp\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0masarray\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m[\u001b[0m\u001b[0mclf\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mpredict_proba\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mX\u001b[0m\u001b[1;33m)\u001b[0m \u001b[1;32mfor\u001b[0m \u001b[0mclf\u001b[0m \u001b[1;32min\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mestimators_\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    325\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    326\u001b[0m     \u001b[1;32mdef\u001b[0m \u001b[0m_predict_proba\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mX\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mD:\\Softwares\\Anaconda3\\envs\\neuroerg\\lib\\site-packages\\sklearn\\ensemble\\_voting.py\u001b[0m in \u001b[0;36m<listcomp>\u001b[1;34m(.0)\u001b[0m\n\u001b[0;32m    322\u001b[0m     \u001b[1;32mdef\u001b[0m \u001b[0m_collect_probas\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mX\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    323\u001b[0m         \u001b[1;34m\"\"\"Collect results from clf.predict calls.\"\"\"\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 324\u001b[1;33m         \u001b[1;32mreturn\u001b[0m \u001b[0mnp\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0masarray\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m[\u001b[0m\u001b[0mclf\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mpredict_proba\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mX\u001b[0m\u001b[1;33m)\u001b[0m \u001b[1;32mfor\u001b[0m \u001b[0mclf\u001b[0m \u001b[1;32min\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mestimators_\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    325\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    326\u001b[0m     \u001b[1;32mdef\u001b[0m \u001b[0m_predict_proba\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mX\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mD:\\Softwares\\Anaconda3\\envs\\neuroerg\\lib\\site-packages\\sklearn\\ensemble\\_voting.py\u001b[0m in \u001b[0;36mpredict_proba\u001b[1;34m(self)\u001b[0m\n\u001b[0;32m    347\u001b[0m         \u001b[1;32mif\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mvoting\u001b[0m \u001b[1;33m==\u001b[0m \u001b[1;34m'hard'\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    348\u001b[0m             raise AttributeError(\"predict_proba is not available when\"\n\u001b[1;32m--> 349\u001b[1;33m                                  \" voting=%r\" % self.voting)\n\u001b[0m\u001b[0;32m    350\u001b[0m         \u001b[1;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_predict_proba\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    351\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;31mAttributeError\u001b[0m: predict_proba is not available when voting='hard'"
     ]
    }
   ],
   "source": [
    "elvl2_s = VC(estimators = [(\"NuSVC_hard\",svocs3),(\"NuSVC_xgb_soft\",ms)],voting='soft',n_jobs=-1)\n",
    "# elvl2_h = VC(estimators = [(\"NuSVC_xgb_soft\",ms),(\"NuSVC_hard\",svocs3)],voting='hard',n_jobs=-1)\n",
    "\n",
    "for sub in range(1,16):\n",
    "    key = \"lvl2_soft\"\n",
    "    X_test,y_test = load_dat(sub,1)\n",
    "    X_train,y_train = load_dat(sub,2)\n",
    "    X_p = load_test_dat(sub,3)\n",
    "    print(sub)\n",
    "\n",
    "    fname = f\"D:/Softwares/AnacondaWorkDir/Neuroergo_Hack/Predictions/{key}_{sub}.csv\"\n",
    "    \n",
    "    start = time.time()\n",
    "    elvl2_s.fit(X_train,y_train)\n",
    "    end = time.time()\n",
    "    t1 = end-start\n",
    "    \n",
    "    start = time.time()\n",
    "    p_train = elvl2_s.predict(X_train)\n",
    "    p_test = elvl2_s.predict(X_test)\n",
    "    end = time.time()\n",
    "    t2 = end-start\n",
    "    \n",
    "    c = np.around(cm(y_test,p_test,normalize='true'),2)\n",
    "    \n",
    "    p_pred = elvl2_s.predict(X_p)\n",
    "    pd.DataFrame(p_pred).to_csv(fname)\n",
    "    \n",
    "    print(f'model {key},Train time = {t1}, Predict time = {t2}, train acc = {accuracy_score(y_train,p_train)}, test acc = {accuracy_score(y_test,p_test)}.')\n",
    "    print(c)\n",
    "    \n",
    "    filename = f\"D:/Softwares/AnacondaWorkDir/Neuroergo_Hack/Predictions/{key}_{sub}.pickle\"\n",
    "    pickle.dump(clf, open(filename, 'wb'))\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "tsck2 = make_pipeline(Covariances(estimator='oas'),tsc(tsupdate=True,clf=sknn(n_neighbors = 14,n_jobs=-1,weights='distance')))\n",
    "svock2 = VC(estimators = [('mdm',mdmc),('Fgmdm',Fgmdmc),('tsc',tsck2)], voting='hard',n_jobs=-1)\n",
    "svock2.fit(X2,y2)\n",
    "pk1 = svock2.predict(X1)\n",
    "pk2 = svock2.predict(X2)\n",
    "print(\"svock2 hard train = \",accuracy_score(y2,pk2),\" test = \",accuracy_score(y1,pk1))\n",
    "ckx = np.around(cm(y1,pk1,normalize='true'),2)\n",
    "ckx"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "tscs3 = make_pipeline(Covariances(estimator='oas'),tsc(tsupdate=True,clf=NuSVC()))\n",
    "svocs3 = VC(estimators = [('mdm',mdmc),('Fgmdm',Fgmdmc),('tsc',tscs3)], voting='hard',n_jobs=-1)\n",
    "svocs3.fit(X2,y2)\n",
    "pn1 = svocs3.predict(X1)\n",
    "pn2 = svocs3.predict(X2)\n",
    "print(\"svocs3 hard train = \",accuracy_score(y2,pn2),\" test = \",accuracy_score(y1,pn1))\n",
    "cnx = np.around(cm(y1,pn1,normalize='true'),2)\n",
    "cnx"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "tscs2 = make_pipeline(Covariances(estimator='oas'),tsc(tsupdate=True,clf=SVC(gamma='auto')))\n",
    "svocs2 = VC(estimators = [('mdm',mdmc),('Fgmdm',Fgmdmc),('tsc',tscs2)], voting='hard',n_jobs=-1)\n",
    "svocs2.fit(X2,y2)\n",
    "ps1 = svocs2.predict(X1)\n",
    "ps2 = svocs2.predict(X2)\n",
    "print(\"svocs2 hard train = \",accuracy_score(y2,ps2),\" test = \",accuracy_score(y1,ps1))\n",
    "csx = np.around(cm(y1,ps1,normalize='true'),2)\n",
    "csx"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "tscx = make_pipeline(Covariances(estimator='oas'),tsc(tsupdate=True,clf=xgb.XGBClassifier(n_estimators = 1500,max_depth = 15,learning_rate= 0.3,reg_lambda = 5,n_jobs=-1)))\n",
    "tscs2 = make_pipeline(Covariances(estimator='oas'),tsc(tsupdate=True,clf=SVC(gamma='auto')))\n",
    "mdmc = make_pipeline(Covariances(estimator='oas'),MDM(n_jobs=-1))\n",
    "sf1 = VC(estimators = [('tsc',tscs2),('tscx',tscx),('mdm',mdmc)],weights = [1,1,2], voting='hard',n_jobs=-1)\n",
    "sf1.fit(X2,y2)\n",
    "p1 = sf1.predict(X1)\n",
    "p2 = sf1.predict(X2)\n",
    "print(\"sf1 hard train = \",accuracy_score(y2,p2),\" test = \",accuracy_score(y1,p1))\n",
    "cf = np.around(cm(y1,p1,normalize='true'),2)\n",
    "cf"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "tscs3 = make_pipeline(Covariances(estimator='oas'),tsc(tsupdate=True,clf=NuSVC(probability=True)))\n",
    "ms = VC(estimators = [('mdm',mdmc),('tscx',tscx),('NuSVC',tscs3)],voting='soft',n_jobs=-1)\n",
    "ms.fit(X2,y2)\n",
    "p1 = ms.predict(X1)\n",
    "p2 = ms.predict(X2)\n",
    "print(\"ms hard train = \",accuracy_score(y2,p2),\" test = \",accuracy_score(y1,p1))\n",
    "\n",
    "cf = np.around(cm(y1,p1,normalize='true'),2)\n",
    "cf"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}