MLP_Ray.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Device is cuda!\n"
     ]
    }
   ],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "from matplotlib import pyplot as plt\n",
    "import seaborn as sns\n",
    "import torch\n",
    "import torch.nn as nn\n",
    "import torch.nn.functional as F\n",
    "import torchvision\n",
    "from torch.utils.data import DataLoader, TensorDataset\n",
    "from sklearn.metrics import precision_score, recall_score\n",
    "from livelossplot import PlotLosses\n",
    "from sklearn.preprocessing import MinMaxScaler, LabelEncoder\n",
    "import os\n",
    "import tempfile\n",
    "import ray\n",
    "from ray import tune, air\n",
    "from ray.tune.schedulers import ASHAScheduler\n",
    "from functools import partial\n",
    "from ray.train import Checkpoint\n",
    "device = torch.device('cuda') if torch.cuda.is_available() else torch.device('cpu')\n",
    "print(f\"Device is {device}!\")\n",
    "\n",
    "local_dir = os.path.abspath(\"./ray_tune\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>dns_qtype</th>\n",
       "      <th>dns_rcode</th>\n",
       "      <th>dns_query</th>\n",
       "      <th>dst_ip_bytes</th>\n",
       "      <th>src_pkts</th>\n",
       "      <th>label</th>\n",
       "      <th>type</th>\n",
       "      <th>conn_state-OTH</th>\n",
       "      <th>conn_state-REJ</th>\n",
       "      <th>conn_state-RSTO</th>\n",
       "      <th>...</th>\n",
       "      <th>service-ssl</th>\n",
       "      <th>dns_AA--1</th>\n",
       "      <th>dns_AA-F</th>\n",
       "      <th>dns_AA-T</th>\n",
       "      <th>dns_RA--1</th>\n",
       "      <th>dns_RA-F</th>\n",
       "      <th>dns_RA-T</th>\n",
       "      <th>dns_RD--1</th>\n",
       "      <th>dns_RD-F</th>\n",
       "      <th>dns_RD-T</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "      <td>3786</td>\n",
       "      <td>6</td>\n",
       "      <td>0</td>\n",
       "      <td>normal</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>12</td>\n",
       "      <td>0</td>\n",
       "      <td>4765</td>\n",
       "      <td>172</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>normal</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>12</td>\n",
       "      <td>3</td>\n",
       "      <td>6112</td>\n",
       "      <td>800</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>normal</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>43</td>\n",
       "      <td>0</td>\n",
       "      <td>5259</td>\n",
       "      <td>525</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>normal</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>43</td>\n",
       "      <td>0</td>\n",
       "      <td>2790</td>\n",
       "      <td>408</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>normal</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>46559</th>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>12284</td>\n",
       "      <td>158</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>normal</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>46560</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "      <td>8462</td>\n",
       "      <td>10</td>\n",
       "      <td>1</td>\n",
       "      <td>ddos</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>46561</th>\n",
       "      <td>12</td>\n",
       "      <td>0</td>\n",
       "      <td>3631</td>\n",
       "      <td>375</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>normal</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>46562</th>\n",
       "      <td>28</td>\n",
       "      <td>0</td>\n",
       "      <td>9338</td>\n",
       "      <td>712</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>normal</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>46563</th>\n",
       "      <td>12</td>\n",
       "      <td>3</td>\n",
       "      <td>6528</td>\n",
       "      <td>349</td>\n",
       "      <td>2</td>\n",
       "      <td>0</td>\n",
       "      <td>normal</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>46564 rows × 45 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "       dns_qtype  dns_rcode  dns_query  dst_ip_bytes  src_pkts  label    type  \\\n",
       "0              0          0          2          3786         6      0  normal   \n",
       "1             12          0       4765           172         1      0  normal   \n",
       "2             12          3       6112           800         1      0  normal   \n",
       "3             43          0       5259           525         1      0  normal   \n",
       "4             43          0       2790           408         1      0  normal   \n",
       "...          ...        ...        ...           ...       ...    ...     ...   \n",
       "46559          1          0      12284           158         1      0  normal   \n",
       "46560          0          0          2          8462        10      1    ddos   \n",
       "46561         12          0       3631           375         1      0  normal   \n",
       "46562         28          0       9338           712         1      0  normal   \n",
       "46563         12          3       6528           349         2      0  normal   \n",
       "\n",
       "       conn_state-OTH  conn_state-REJ  conn_state-RSTO  ...  service-ssl  \\\n",
       "0                   0               0                0  ...            0   \n",
       "1                   0               0                0  ...            0   \n",
       "2                   0               0                0  ...            0   \n",
       "3                   0               0                0  ...            0   \n",
       "4                   0               0                0  ...            0   \n",
       "...               ...             ...              ...  ...          ...   \n",
       "46559               0               0                0  ...            0   \n",
       "46560               0               0                0  ...            1   \n",
       "46561               0               0                0  ...            0   \n",
       "46562               0               0                0  ...            0   \n",
       "46563               0               0                0  ...            0   \n",
       "\n",
       "       dns_AA--1  dns_AA-F  dns_AA-T  dns_RA--1  dns_RA-F  dns_RA-T  \\\n",
       "0              1         0         0          1         0         0   \n",
       "1              0         0         1          0         0         1   \n",
       "2              0         1         0          0         1         0   \n",
       "3              0         1         0          0         1         0   \n",
       "4              0         1         0          0         1         0   \n",
       "...          ...       ...       ...        ...       ...       ...   \n",
       "46559          0         0         1          0         1         0   \n",
       "46560          1         0         0          1         0         0   \n",
       "46561          0         1         0          0         1         0   \n",
       "46562          0         1         0          0         1         0   \n",
       "46563          0         1         0          0         1         0   \n",
       "\n",
       "       dns_RD--1  dns_RD-F  dns_RD-T  \n",
       "0              1         0         0  \n",
       "1              0         1         0  \n",
       "2              0         1         0  \n",
       "3              0         1         0  \n",
       "4              0         1         0  \n",
       "...          ...       ...       ...  \n",
       "46559          0         1         0  \n",
       "46560          1         0         0  \n",
       "46561          0         1         0  \n",
       "46562          0         1         0  \n",
       "46563          0         0         1  \n",
       "\n",
       "[46564 rows x 45 columns]"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df = pd.read_csv('./preprocessed.csv', low_memory=False) \n",
    "df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>dns_qtype</th>\n",
       "      <th>dns_rcode</th>\n",
       "      <th>dns_query</th>\n",
       "      <th>dst_ip_bytes</th>\n",
       "      <th>src_pkts</th>\n",
       "      <th>conn_state-OTH</th>\n",
       "      <th>conn_state-REJ</th>\n",
       "      <th>conn_state-RSTO</th>\n",
       "      <th>conn_state-RSTOS0</th>\n",
       "      <th>conn_state-RSTR</th>\n",
       "      <th>...</th>\n",
       "      <th>service-ssl</th>\n",
       "      <th>dns_AA--1</th>\n",
       "      <th>dns_AA-F</th>\n",
       "      <th>dns_AA-T</th>\n",
       "      <th>dns_RA--1</th>\n",
       "      <th>dns_RA-F</th>\n",
       "      <th>dns_RA-T</th>\n",
       "      <th>dns_RD--1</th>\n",
       "      <th>dns_RD-F</th>\n",
       "      <th>dns_RD-T</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "      <td>3786</td>\n",
       "      <td>6</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>12</td>\n",
       "      <td>0</td>\n",
       "      <td>4765</td>\n",
       "      <td>172</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>12</td>\n",
       "      <td>3</td>\n",
       "      <td>6112</td>\n",
       "      <td>800</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>43</td>\n",
       "      <td>0</td>\n",
       "      <td>5259</td>\n",
       "      <td>525</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>43</td>\n",
       "      <td>0</td>\n",
       "      <td>2790</td>\n",
       "      <td>408</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 43 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "   dns_qtype  dns_rcode  dns_query  dst_ip_bytes  src_pkts  conn_state-OTH  \\\n",
       "0          0          0          2          3786         6               0   \n",
       "1         12          0       4765           172         1               0   \n",
       "2         12          3       6112           800         1               0   \n",
       "3         43          0       5259           525         1               0   \n",
       "4         43          0       2790           408         1               0   \n",
       "\n",
       "   conn_state-REJ  conn_state-RSTO  conn_state-RSTOS0  conn_state-RSTR  ...  \\\n",
       "0               0                0                  0                0  ...   \n",
       "1               0                0                  0                0  ...   \n",
       "2               0                0                  0                0  ...   \n",
       "3               0                0                  0                0  ...   \n",
       "4               0                0                  0                0  ...   \n",
       "\n",
       "   service-ssl  dns_AA--1  dns_AA-F  dns_AA-T  dns_RA--1  dns_RA-F  dns_RA-T  \\\n",
       "0            0          1         0         0          1         0         0   \n",
       "1            0          0         0         1          0         0         1   \n",
       "2            0          0         1         0          0         1         0   \n",
       "3            0          0         1         0          0         1         0   \n",
       "4            0          0         1         0          0         1         0   \n",
       "\n",
       "   dns_RD--1  dns_RD-F  dns_RD-T  \n",
       "0          1         0         0  \n",
       "1          0         1         0  \n",
       "2          0         1         0  \n",
       "3          0         1         0  \n",
       "4          0         1         0  \n",
       "\n",
       "[5 rows x 43 columns]"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X = df.drop(['label','type'], axis=1)\n",
    "X.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>label</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   label\n",
       "0      0\n",
       "1      0\n",
       "2      0\n",
       "3      0\n",
       "4      0"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "y = df.filter(items=['label'])\n",
    "y.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>0</th>\n",
       "      <th>1</th>\n",
       "      <th>2</th>\n",
       "      <th>3</th>\n",
       "      <th>4</th>\n",
       "      <th>5</th>\n",
       "      <th>6</th>\n",
       "      <th>7</th>\n",
       "      <th>8</th>\n",
       "      <th>9</th>\n",
       "      <th>...</th>\n",
       "      <th>33</th>\n",
       "      <th>34</th>\n",
       "      <th>35</th>\n",
       "      <th>36</th>\n",
       "      <th>37</th>\n",
       "      <th>38</th>\n",
       "      <th>39</th>\n",
       "      <th>40</th>\n",
       "      <th>41</th>\n",
       "      <th>42</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000141</td>\n",
       "      <td>0.000044</td>\n",
       "      <td>0.000024</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0.047059</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.336797</td>\n",
       "      <td>0.000002</td>\n",
       "      <td>0.000004</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0.047059</td>\n",
       "      <td>0.6</td>\n",
       "      <td>0.432005</td>\n",
       "      <td>0.000009</td>\n",
       "      <td>0.000004</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>0.168627</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.371713</td>\n",
       "      <td>0.000006</td>\n",
       "      <td>0.000004</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0.168627</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.197201</td>\n",
       "      <td>0.000005</td>\n",
       "      <td>0.000004</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 43 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "         0    1         2         3         4    5    6    7    8    9   ...  \\\n",
       "0  0.000000  0.0  0.000141  0.000044  0.000024  0.0  0.0  0.0  0.0  0.0  ...   \n",
       "1  0.047059  0.0  0.336797  0.000002  0.000004  0.0  0.0  0.0  0.0  0.0  ...   \n",
       "2  0.047059  0.6  0.432005  0.000009  0.000004  0.0  0.0  0.0  0.0  0.0  ...   \n",
       "3  0.168627  0.0  0.371713  0.000006  0.000004  0.0  0.0  0.0  0.0  0.0  ...   \n",
       "4  0.168627  0.0  0.197201  0.000005  0.000004  0.0  0.0  0.0  0.0  0.0  ...   \n",
       "\n",
       "    33   34   35   36   37   38   39   40   41   42  \n",
       "0  0.0  1.0  0.0  0.0  1.0  0.0  0.0  1.0  0.0  0.0  \n",
       "1  0.0  0.0  0.0  1.0  0.0  0.0  1.0  0.0  1.0  0.0  \n",
       "2  0.0  0.0  1.0  0.0  0.0  1.0  0.0  0.0  1.0  0.0  \n",
       "3  0.0  0.0  1.0  0.0  0.0  1.0  0.0  0.0  1.0  0.0  \n",
       "4  0.0  0.0  1.0  0.0  0.0  1.0  0.0  0.0  1.0  0.0  \n",
       "\n",
       "[5 rows x 43 columns]"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "min_max_scaler = MinMaxScaler()\n",
    "X =  pd.DataFrame(min_max_scaler.fit_transform(X))\n",
    "X.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "x = torch.Tensor(X.values) \n",
    "gt = torch.Tensor(y.values)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "torch.Size([46564, 1])"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "gt.size()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "torch.Size([46564, 43])"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "x.size()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "#torch.manual_seed(42)\n",
    "\n",
    "# Create a TensorDataset from your data and labels\n",
    "dataset = TensorDataset(x, gt)\n",
    "\n",
    "# Calculate the number of samples for training and testing datasets\n",
    "num_samples = len(dataset)\n",
    "train_size = int(num_samples * 0.7)\n",
    "test_size = num_samples - train_size\n",
    "\n",
    "def load_data():\n",
    "    train_dataset, test_dataset = torch.utils.data.random_split(dataset, [train_size, test_size])\n",
    "    return train_dataset, test_dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "class MLP(nn.Module):\n",
    "    def __init__(self, l1=100, l2=128, l3=256):\n",
    "        super(MLP, self).__init__()\n",
    "        self.l1 = nn.Linear(43, l1)\n",
    "        self.b1 = nn.BatchNorm1d(l1)\n",
    "        self.l2 = nn.Linear(l1, l2)\n",
    "        self.b2 = nn.BatchNorm1d(l2)\n",
    "        self.l3 = nn.Linear(l2, l3)\n",
    "        self.b3 = nn.BatchNorm1d(l3)\n",
    "        self.last = nn.Linear(l3, 1)\n",
    "\n",
    "    def forward(self, x):\n",
    "        x = torch.relu(self.b1(self.l1(x)))\n",
    "        x = torch.relu(self.b2(self.l2(x)))\n",
    "        x = torch.relu(self.b3(self.l3(x)))\n",
    "        return torch.sigmoid(self.last(x))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "def train_mlp(config):\n",
    "    net = MLP(config[\"l1\"], config[\"l2\"], config[\"l3\"]).to(device)\n",
    "\n",
    "    criterion = nn.BCELoss()\n",
    "    optimizer = torch.optim.Adam(net.parameters(), lr=config[\"lr\"])\n",
    "\n",
    "    if \"restore\" in config:\n",
    "        checkpoint_path = config[\"restore\"]\n",
    "        checkpoint = torch.load(checkpoint_path)\n",
    "        net.load_state_dict(checkpoint[\"model_state_dict\"])\n",
    "        optimizer.load_state_dict(checkpoint[\"optimizer_state_dict\"])\n",
    "\n",
    "    trainset, _ = load_data()\n",
    "\n",
    "    test_abs = int(len(trainset) * 0.7)\n",
    "    train_subset, val_subset = torch.utils.data.random_split(\n",
    "        trainset, [test_abs, len(trainset) - test_abs])\n",
    "    \n",
    "    # Create DataLoaders\n",
    "    trainloader = DataLoader(train_subset, batch_size=int(config[\"batch_size\"]), shuffle=True, num_workers=0)\n",
    "    valloader = DataLoader(val_subset, batch_size=int(config[\"batch_size\"]), shuffle=True, num_workers=0)\n",
    "\n",
    "    for epoch in range(100):  # loop over the dataset multiple times\n",
    "        running_loss = 0.0\n",
    "        epoch_steps = 0\n",
    "        for i, data in enumerate(trainloader):\n",
    "            inputs, labels = data\n",
    "            inputs, labels = inputs.to(device), labels.to(device)\n",
    "            optimizer.zero_grad()\n",
    "            outputs = net(inputs)\n",
    "            loss = criterion(outputs, labels)\n",
    "            loss.backward()\n",
    "            optimizer.step()\n",
    "            running_loss += loss.item()\n",
    "            epoch_steps += 1\n",
    "\n",
    "        val_loss = 0.0\n",
    "        val_steps = 0\n",
    "        total_correct = 0\n",
    "        total_samples = 0\n",
    "        for i, data in enumerate(valloader, 0):\n",
    "            with torch.no_grad():\n",
    "                inputs, labels = data\n",
    "                inputs, labels = inputs.to(device), labels.to(device)\n",
    "                outputs = net(inputs)\n",
    "                preds = outputs.round()\n",
    "                total_correct += preds.eq(labels).sum().item()\n",
    "                total_samples += labels.size(0)\n",
    "                loss = criterion(outputs, labels)\n",
    "                val_loss += loss.cpu().numpy()\n",
    "                val_steps += 1\n",
    "\n",
    "        # Calculate overall validation accuracy\n",
    "        accuracy = (total_correct / total_samples) * 100\n",
    "                \n",
    "        with tempfile.TemporaryDirectory() as temp_checkpoint_dir:\n",
    "            checkpoint_path = os.path.join(temp_checkpoint_dir, \"checkpoint.pt\")\n",
    "            torch.save({\n",
    "                \"model_state_dict\": net.state_dict(),\n",
    "                \"optimizer_state_dict\": optimizer.state_dict(),\n",
    "            }, checkpoint_path)\n",
    "\n",
    "            # Use ray.train.report to report metrics and checkpoint\n",
    "            ray.train.report({\n",
    "                \"loss\": val_loss,\n",
    "                \"accuracy\": accuracy\n",
    "            }, checkpoint=Checkpoint.from_directory(temp_checkpoint_dir))\n",
    "    \n",
    "    print(\"Finished Training\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "def test_best_model(best_result):\n",
    "    best_trained_model = MLP(best_result.config[\"l1\"], best_result.config[\"l2\"], best_result.config[\"l3\"]).to(device)\n",
    "    \n",
    "    print(f'> Number of parameters {len(torch.nn.utils.parameters_to_vector(best_trained_model.parameters()))}')\n",
    "\n",
    "    checkpoint_path = os.path.join(best_result.checkpoint.to_directory(), \"checkpoint.pt\")\n",
    "\n",
    "    # Load the checkpoint\n",
    "    checkpoint = torch.load(checkpoint_path)\n",
    "    best_trained_model.load_state_dict(checkpoint['model_state_dict'])\n",
    "\n",
    "    _, testset = load_data()\n",
    "    \n",
    "    testloader = DataLoader(testset, batch_size=4, shuffle=False, num_workers=2)\n",
    "\n",
    "    correct = 0\n",
    "    total = 0\n",
    "    with torch.no_grad():\n",
    "        for data in testloader:\n",
    "            inputs, labels = data\n",
    "            inputs, labels = inputs.to(device), labels.to(device)\n",
    "            outputs = best_trained_model(inputs)\n",
    "            predicted = outputs.round()\n",
    "            total += labels.size(0)\n",
    "            correct += (predicted == labels).sum().item()\n",
    "\n",
    "    accuracy = 100 * correct / total\n",
    "    print(f\"Best trial test set accuracy: {accuracy}%\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div class=\"tuneStatus\">\n",
       "  <div style=\"display: flex;flex-direction: row\">\n",
       "    <div style=\"display: flex;flex-direction: column;\">\n",
       "      <h3>Tune Status</h3>\n",
       "      <table>\n",
       "<tbody>\n",
       "<tr><td>Current time:</td><td>2024-02-22 18:16:33</td></tr>\n",
       "<tr><td>Running for: </td><td>00:02:21.00        </td></tr>\n",
       "<tr><td>Memory:      </td><td>131.4/1132.4 GiB   </td></tr>\n",
       "</tbody>\n",
       "</table>\n",
       "    </div>\n",
       "    <div class=\"vDivider\"></div>\n",
       "    <div class=\"systemInfo\">\n",
       "      <h3>System Info</h3>\n",
       "      Using AsyncHyperBand: num_stopped=10<br>Bracket: Iter 64.000: -1.7695383168756962 | Iter 32.000: -1.7162782214581966 | Iter 16.000: -1.7399756629019976 | Iter 8.000: -2.878801554441452 | Iter 4.000: -4.443135190755129 | Iter 2.000: -5.029785729944706 | Iter 1.000: -7.2969324216246605<br>Logical resource usage: 64.0/64 CPUs, 1.0/1 GPUs (0.0/1.0 accelerator_type:A100)\n",
       "    </div>\n",
       "    \n",
       "  </div>\n",
       "  <div class=\"hDivider\"></div>\n",
       "  <div class=\"trialStatus\">\n",
       "    <h3>Trial Status</h3>\n",
       "    <table>\n",
       "<thead>\n",
       "<tr><th>Trial name           </th><th>status    </th><th>loc                 </th><th style=\"text-align: right;\">  batch_size</th><th style=\"text-align: right;\">         lr</th><th style=\"text-align: right;\">  iter</th><th style=\"text-align: right;\">  total time (s)</th><th style=\"text-align: right;\">    loss</th><th style=\"text-align: right;\">  accuracy</th></tr>\n",
       "</thead>\n",
       "<tbody>\n",
       "<tr><td>train_mlp_2dc03_00000</td><td>TERMINATED</td><td>10.234.204.81:149878</td><td style=\"text-align: right;\">         512</td><td style=\"text-align: right;\">0.000797296</td><td style=\"text-align: right;\">   100</td><td style=\"text-align: right;\">       57.1088  </td><td style=\"text-align: right;\"> 2.36964</td><td style=\"text-align: right;\">   94.6518</td></tr>\n",
       "<tr><td>train_mlp_2dc03_00001</td><td>TERMINATED</td><td>10.234.204.81:149878</td><td style=\"text-align: right;\">         256</td><td style=\"text-align: right;\">0.000221477</td><td style=\"text-align: right;\">     1</td><td style=\"text-align: right;\">        0.62322 </td><td style=\"text-align: right;\">15.7761 </td><td style=\"text-align: right;\">   91.8294</td></tr>\n",
       "<tr><td>train_mlp_2dc03_00002</td><td>TERMINATED</td><td>10.234.204.81:149878</td><td style=\"text-align: right;\">        1024</td><td style=\"text-align: right;\">6.85912e-05</td><td style=\"text-align: right;\">     4</td><td style=\"text-align: right;\">        2.03971 </td><td style=\"text-align: right;\"> 4.78391</td><td style=\"text-align: right;\">   90.5716</td></tr>\n",
       "<tr><td>train_mlp_2dc03_00003</td><td>TERMINATED</td><td>10.234.204.81:149878</td><td style=\"text-align: right;\">         256</td><td style=\"text-align: right;\">0.0172756  </td><td style=\"text-align: right;\">     8</td><td style=\"text-align: right;\">        5.20675 </td><td style=\"text-align: right;\"> 4.32751</td><td style=\"text-align: right;\">   94.028 </td></tr>\n",
       "<tr><td>train_mlp_2dc03_00004</td><td>TERMINATED</td><td>10.234.204.81:149878</td><td style=\"text-align: right;\">         256</td><td style=\"text-align: right;\">0.00177855 </td><td style=\"text-align: right;\">     4</td><td style=\"text-align: right;\">        2.65913 </td><td style=\"text-align: right;\"> 4.78917</td><td style=\"text-align: right;\">   93.8133</td></tr>\n",
       "<tr><td>train_mlp_2dc03_00005</td><td>TERMINATED</td><td>10.234.204.81:149878</td><td style=\"text-align: right;\">         128</td><td style=\"text-align: right;\">0.0491863  </td><td style=\"text-align: right;\">     1</td><td style=\"text-align: right;\">        0.891682</td><td style=\"text-align: right;\"> 9.30559</td><td style=\"text-align: right;\">   93.9973</td></tr>\n",
       "<tr><td>train_mlp_2dc03_00006</td><td>TERMINATED</td><td>10.234.204.81:149878</td><td style=\"text-align: right;\">         256</td><td style=\"text-align: right;\">0.000301439</td><td style=\"text-align: right;\">     4</td><td style=\"text-align: right;\">        2.83272 </td><td style=\"text-align: right;\"> 4.56824</td><td style=\"text-align: right;\">   93.9667</td></tr>\n",
       "<tr><td>train_mlp_2dc03_00007</td><td>TERMINATED</td><td>10.234.204.81:149878</td><td style=\"text-align: right;\">          64</td><td style=\"text-align: right;\">0.000256532</td><td style=\"text-align: right;\">     1</td><td style=\"text-align: right;\">        1.42697 </td><td style=\"text-align: right;\">25.457  </td><td style=\"text-align: right;\">   93.4758</td></tr>\n",
       "<tr><td>train_mlp_2dc03_00008</td><td>TERMINATED</td><td>10.234.204.81:149878</td><td style=\"text-align: right;\">         128</td><td style=\"text-align: right;\">0.000129812</td><td style=\"text-align: right;\">     1</td><td style=\"text-align: right;\">        0.951082</td><td style=\"text-align: right;\">10.6436 </td><td style=\"text-align: right;\">   94.0076</td></tr>\n",
       "<tr><td>train_mlp_2dc03_00009</td><td>TERMINATED</td><td>10.234.204.81:149878</td><td style=\"text-align: right;\">        1024</td><td style=\"text-align: right;\">0.00843087 </td><td style=\"text-align: right;\">   100</td><td style=\"text-align: right;\">       50.4987  </td><td style=\"text-align: right;\"> 1.13423</td><td style=\"text-align: right;\">   94.2939</td></tr>\n",
       "</tbody>\n",
       "</table>\n",
       "  </div>\n",
       "</div>\n",
       "<style>\n",
       ".tuneStatus {\n",
       "  color: var(--jp-ui-font-color1);\n",
       "}\n",
       ".tuneStatus .systemInfo {\n",
       "  display: flex;\n",
       "  flex-direction: column;\n",
       "}\n",
       ".tuneStatus td {\n",
       "  white-space: nowrap;\n",
       "}\n",
       ".tuneStatus .trialStatus {\n",
       "  display: flex;\n",
       "  flex-direction: column;\n",
       "}\n",
       ".tuneStatus h3 {\n",
       "  font-weight: bold;\n",
       "}\n",
       ".tuneStatus .hDivider {\n",
       "  border-bottom-width: var(--jp-border-width);\n",
       "  border-bottom-color: var(--jp-border-color0);\n",
       "  border-bottom-style: solid;\n",
       "}\n",
       ".tuneStatus .vDivider {\n",
       "  border-left-width: var(--jp-border-width);\n",
       "  border-left-color: var(--jp-border-color0);\n",
       "  border-left-style: solid;\n",
       "  margin: 0.5em 1em 0.5em 1em;\n",
       "}\n",
       "</style>\n"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2024-02-22 18:14:22,083\tWARNING worker.py:2065 -- Warning: The actor ImplicitFunc is very large (15 MiB). Check that its definition is not implicitly capturing a large array or other object in scope. Tip: use ray.put() to put large objects in the Ray object store.\n",
      "2024-02-22 18:16:33,737\tINFO tune.py:1154 -- Total run time: 142.81 seconds (140.98 seconds for the tuning loop).\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Best trial config: {'l1': 128, 'l2': 16, 'l3': 64, 'lr': 0.008430868939509882, 'batch_size': 1024}\n",
      "Best trial final validation loss: 1.1342255622148514\n",
      "Best trial final validation accuracy: 94.29389508129665\n",
      "> Number of parameters 9265\n",
      "Best trial test set accuracy: 89.84967788117395%\n"
     ]
    }
   ],
   "source": [
    "def main(num_samples=10, max_num_epochs=100, gpus_per_trial=1):\n",
    "    config = {\n",
    "        \"l1\": tune.sample_from(lambda _: 2 ** np.random.randint(3, 10)),\n",
    "        \"l2\": tune.sample_from(lambda _: 2 ** np.random.randint(3, 10)),\n",
    "        \"l3\": tune.sample_from(lambda _: 2 ** np.random.randint(3, 10)),\n",
    "        \"lr\": tune.loguniform(1e-5, 1e-1),\n",
    "        \"batch_size\": tune.choice([64, 128, 256, 512, 1024]),\n",
    "    }\n",
    "    scheduler = ASHAScheduler(\n",
    "        max_t=max_num_epochs,\n",
    "        grace_period=1,\n",
    "        reduction_factor=2)\n",
    "    \n",
    "    ray.shutdown()\n",
    "    ray.init(log_to_driver=False)\n",
    "    \n",
    "    tuner = tune.Tuner(\n",
    "        tune.with_resources(\n",
    "            tune.with_parameters(train_mlp),\n",
    "            resources={\"cpu\": 64, \"gpu\": gpus_per_trial}\n",
    "        ),\n",
    "        tune_config=tune.TuneConfig(\n",
    "            metric=\"loss\",\n",
    "            mode=\"min\",\n",
    "            scheduler=scheduler,\n",
    "            num_samples=num_samples,\n",
    "        ),\n",
    "        param_space=config,\n",
    "        run_config=air.RunConfig(\n",
    "            local_dir=local_dir,\n",
    "            name=\"mlp\",\n",
    "        )\n",
    "    )\n",
    "    results = tuner.fit()\n",
    "    \n",
    "    best_result = results.get_best_result(\"loss\", \"min\")\n",
    "\n",
    "    print(\"Best trial config: {}\".format(best_result.config))\n",
    "    print(\"Best trial final validation loss: {}\".format(\n",
    "        best_result.metrics[\"loss\"]))\n",
    "    print(\"Best trial final validation accuracy: {}\".format(\n",
    "        best_result.metrics[\"accuracy\"]))\n",
    "\n",
    "    test_best_model(best_result)\n",
    "    \n",
    "main(num_samples=10, max_num_epochs=100, gpus_per_trial=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "10"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "y = df.filter(items=['type'])\n",
    "y['type'].nunique()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home2/tghv73/myjupyterenv/lib/python3.8/site-packages/sklearn/preprocessing/_label.py:114: DataConversionWarning: A column-vector y was passed when a 1d array was expected. Please change the shape of y to (n_samples, ), for example using ravel().\n",
      "  y = column_or_1d(y, warn=True)\n"
     ]
    }
   ],
   "source": [
    "label_encoder = LabelEncoder()\n",
    "\n",
    "y_int = label_encoder.fit_transform(y)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "x = torch.Tensor(X.values) \n",
    "gt = torch.Tensor(y_int).long()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "#torch.manual_seed(42)\n",
    "\n",
    "# Create a TensorDataset from your data and labels\n",
    "dataset = TensorDataset(x, gt)\n",
    "\n",
    "# Calculate the number of samples for training and testing datasets\n",
    "num_samples = len(dataset)\n",
    "train_size = int(num_samples * 0.7)\n",
    "test_size = num_samples - train_size"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "class MLP(nn.Module):\n",
    "    def __init__(self, l1=100, l2=128, l3=256):\n",
    "        super(MLP, self).__init__()\n",
    "        self.l1 = nn.Linear(43, l1)\n",
    "        self.b1 = nn.BatchNorm1d(l1)\n",
    "        self.l2 = nn.Linear(l1, l2)\n",
    "        self.b2 = nn.BatchNorm1d(l2)\n",
    "        self.l3 = nn.Linear(l2, l3)\n",
    "        self.b3 = nn.BatchNorm1d(l3)\n",
    "        self.last = nn.Linear(l3, 10)\n",
    "\n",
    "    def forward(self, x):\n",
    "        x = torch.relu(self.b1(self.l1(x)))\n",
    "        x = torch.relu(self.b2(self.l2(x)))\n",
    "        x = torch.relu(self.b3(self.l3(x)))\n",
    "        return torch.softmax(self.last(x), dim=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "def train_mlp(config):\n",
    "    net = MLP(config[\"l1\"], config[\"l2\"], config[\"l3\"]).to(device)\n",
    "\n",
    "    criterion = nn.CrossEntropyLoss()\n",
    "    optimizer = torch.optim.Adam(net.parameters(), lr=config[\"lr\"])\n",
    "\n",
    "    if \"restore\" in config:\n",
    "        checkpoint_path = config[\"restore\"]\n",
    "        checkpoint = torch.load(checkpoint_path)\n",
    "        net.load_state_dict(checkpoint[\"model_state_dict\"])\n",
    "        optimizer.load_state_dict(checkpoint[\"optimizer_state_dict\"])\n",
    "\n",
    "    trainset, _ = load_data()\n",
    "\n",
    "    test_abs = int(len(trainset) * 0.7)\n",
    "    train_subset, val_subset = torch.utils.data.random_split(\n",
    "        trainset, [test_abs, len(trainset) - test_abs])\n",
    "    \n",
    "    # Create DataLoaders\n",
    "    trainloader = DataLoader(train_subset, batch_size=int(config[\"batch_size\"]), shuffle=True, num_workers=0)\n",
    "    valloader = DataLoader(val_subset, batch_size=int(config[\"batch_size\"]), shuffle=True, num_workers=0)\n",
    "\n",
    "    for epoch in range(100):  # loop over the dataset multiple times\n",
    "        running_loss = 0.0\n",
    "        epoch_steps = 0\n",
    "        for i, data in enumerate(trainloader):\n",
    "            inputs, labels = data\n",
    "            inputs, labels = inputs.to(device), labels.to(device)\n",
    "            optimizer.zero_grad()\n",
    "            outputs = net(inputs)\n",
    "            loss = criterion(outputs, labels)\n",
    "            loss.backward()\n",
    "            optimizer.step()\n",
    "            running_loss += loss.item()\n",
    "            epoch_steps += 1\n",
    "\n",
    "        val_loss = 0.0\n",
    "        val_steps = 0\n",
    "        total_correct = 0\n",
    "        total_samples = 0\n",
    "        for i, data in enumerate(valloader, 0):\n",
    "            with torch.no_grad():\n",
    "                inputs, labels = data\n",
    "                inputs, labels = inputs.to(device), labels.to(device)\n",
    "                outputs = net(inputs)\n",
    "                _, argmax = torch.max(outputs, dim=1)\n",
    "                total_correct += argmax.eq(labels).sum().item()\n",
    "                total_samples += labels.size(0)\n",
    "                loss = criterion(outputs, labels)\n",
    "                val_loss += loss.cpu().numpy()\n",
    "                val_steps += 1\n",
    "\n",
    "        # Calculate overall validation accuracy\n",
    "        accuracy = (total_correct / total_samples) * 100\n",
    "                \n",
    "        with tempfile.TemporaryDirectory() as temp_checkpoint_dir:\n",
    "            checkpoint_path = os.path.join(temp_checkpoint_dir, \"checkpoint.pt\")\n",
    "            torch.save({\n",
    "                \"model_state_dict\": net.state_dict(),\n",
    "                \"optimizer_state_dict\": optimizer.state_dict(),\n",
    "            }, checkpoint_path)\n",
    "\n",
    "            # Use ray.train.report to report metrics and checkpoint\n",
    "            ray.train.report({\n",
    "                \"loss\": val_loss,\n",
    "                \"accuracy\": accuracy\n",
    "            }, checkpoint=Checkpoint.from_directory(temp_checkpoint_dir))\n",
    "    \n",
    "    print(\"Finished Training\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "def test_best_model(best_result):\n",
    "    best_trained_model = MLP(best_result.config[\"l1\"], best_result.config[\"l2\"], best_result.config[\"l3\"]).to(device)\n",
    "    \n",
    "    print(f'> Number of parameters {len(torch.nn.utils.parameters_to_vector(best_trained_model.parameters()))}')\n",
    "\n",
    "    checkpoint_path = os.path.join(best_result.checkpoint.to_directory(), \"checkpoint.pt\")\n",
    "\n",
    "    # Load the checkpoint\n",
    "    checkpoint = torch.load(checkpoint_path)\n",
    "    best_trained_model.load_state_dict(checkpoint['model_state_dict'])\n",
    "\n",
    "    _, testset = load_data()\n",
    "    \n",
    "    testloader = DataLoader(testset, batch_size=4, shuffle=False, num_workers=2)\n",
    "\n",
    "    correct = 0\n",
    "    total = 0\n",
    "    with torch.no_grad():\n",
    "        for data in testloader:\n",
    "            inputs, labels = data\n",
    "            inputs, labels = inputs.to(device), labels.to(device)\n",
    "            outputs = best_trained_model(inputs)\n",
    "            _, predicted = torch.max(outputs, dim=1)\n",
    "            total += labels.size(0)\n",
    "            correct += (predicted == labels).sum().item()\n",
    "\n",
    "    accuracy = 100 * correct / total\n",
    "    print(f\"Best trial test set accuracy: {accuracy}%\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div class=\"tuneStatus\">\n",
       "  <div style=\"display: flex;flex-direction: row\">\n",
       "    <div style=\"display: flex;flex-direction: column;\">\n",
       "      <h3>Tune Status</h3>\n",
       "      <table>\n",
       "<tbody>\n",
       "<tr><td>Current time:</td><td>2024-03-04 19:57:50</td></tr>\n",
       "<tr><td>Running for: </td><td>00:04:00.83        </td></tr>\n",
       "<tr><td>Memory:      </td><td>52.6/1132.4 GiB    </td></tr>\n",
       "</tbody>\n",
       "</table>\n",
       "    </div>\n",
       "    <div class=\"vDivider\"></div>\n",
       "    <div class=\"systemInfo\">\n",
       "      <h3>System Info</h3>\n",
       "      Using AsyncHyperBand: num_stopped=20<br>Bracket: Iter 64.000: -30.76240849494934 | Iter 32.000: -30.853620529174805 | Iter 16.000: -30.78410053253174 | Iter 8.000: -30.765214204788208 | Iter 4.000: -30.837124824523926 | Iter 2.000: -30.91453719139099 | Iter 1.000: -45.0549533367157<br>Logical resource usage: 64.0/64 CPUs, 1.0/1 GPUs (0.0/1.0 accelerator_type:A100)\n",
       "    </div>\n",
       "    \n",
       "  </div>\n",
       "  <div class=\"hDivider\"></div>\n",
       "  <div class=\"trialStatus\">\n",
       "    <h3>Trial Status</h3>\n",
       "    <table>\n",
       "<thead>\n",
       "<tr><th>Trial name           </th><th>status    </th><th>loc                  </th><th style=\"text-align: right;\">  batch_size</th><th style=\"text-align: right;\">         lr</th><th style=\"text-align: right;\">  iter</th><th style=\"text-align: right;\">  total time (s)</th><th style=\"text-align: right;\">    loss</th><th style=\"text-align: right;\">  accuracy</th></tr>\n",
       "</thead>\n",
       "<tbody>\n",
       "<tr><td>train_mlp_eb0f3_00000</td><td>TERMINATED</td><td>10.234.204.79:3157852</td><td style=\"text-align: right;\">         512</td><td style=\"text-align: right;\">0.000103081</td><td style=\"text-align: right;\">   100</td><td style=\"text-align: right;\">       35.6843  </td><td style=\"text-align: right;\"> 30.9507</td><td style=\"text-align: right;\">   92.0237</td></tr>\n",
       "<tr><td>train_mlp_eb0f3_00001</td><td>TERMINATED</td><td>10.234.204.79:3157852</td><td style=\"text-align: right;\">         512</td><td style=\"text-align: right;\">0.00410078 </td><td style=\"text-align: right;\">   100</td><td style=\"text-align: right;\">       33.5567  </td><td style=\"text-align: right;\"> 31.3052</td><td style=\"text-align: right;\">   89.5695</td></tr>\n",
       "<tr><td>train_mlp_eb0f3_00002</td><td>TERMINATED</td><td>10.234.204.79:3157852</td><td style=\"text-align: right;\">          64</td><td style=\"text-align: right;\">0.000862208</td><td style=\"text-align: right;\">     1</td><td style=\"text-align: right;\">        1.07395 </td><td style=\"text-align: right;\">235.363 </td><td style=\"text-align: right;\">   92.3714</td></tr>\n",
       "<tr><td>train_mlp_eb0f3_00003</td><td>TERMINATED</td><td>10.234.204.79:3157852</td><td style=\"text-align: right;\">         512</td><td style=\"text-align: right;\">2.52535e-05</td><td style=\"text-align: right;\">     1</td><td style=\"text-align: right;\">        0.337061</td><td style=\"text-align: right;\"> 45.6455</td><td style=\"text-align: right;\">   18.8976</td></tr>\n",
       "<tr><td>train_mlp_eb0f3_00004</td><td>TERMINATED</td><td>10.234.204.79:3157852</td><td style=\"text-align: right;\">         512</td><td style=\"text-align: right;\">0.0278299  </td><td style=\"text-align: right;\">    64</td><td style=\"text-align: right;\">       24.0057  </td><td style=\"text-align: right;\"> 31.9873</td><td style=\"text-align: right;\">   86.2665</td></tr>\n",
       "<tr><td>train_mlp_eb0f3_00005</td><td>TERMINATED</td><td>10.234.204.79:3157852</td><td style=\"text-align: right;\">         512</td><td style=\"text-align: right;\">0.00321301 </td><td style=\"text-align: right;\">   100</td><td style=\"text-align: right;\">       32.3937  </td><td style=\"text-align: right;\"> 30.7799</td><td style=\"text-align: right;\">   92.218 </td></tr>\n",
       "<tr><td>train_mlp_eb0f3_00006</td><td>TERMINATED</td><td>10.234.204.79:3157852</td><td style=\"text-align: right;\">         512</td><td style=\"text-align: right;\">0.080678   </td><td style=\"text-align: right;\">     2</td><td style=\"text-align: right;\">        0.744196</td><td style=\"text-align: right;\"> 31.9288</td><td style=\"text-align: right;\">   86.6653</td></tr>\n",
       "<tr><td>train_mlp_eb0f3_00007</td><td>TERMINATED</td><td>10.234.204.79:3157852</td><td style=\"text-align: right;\">          64</td><td style=\"text-align: right;\">0.0886022  </td><td style=\"text-align: right;\">     1</td><td style=\"text-align: right;\">        1.01313 </td><td style=\"text-align: right;\">243.97  </td><td style=\"text-align: right;\">   86.6449</td></tr>\n",
       "<tr><td>train_mlp_eb0f3_00008</td><td>TERMINATED</td><td>10.234.204.79:3157852</td><td style=\"text-align: right;\">         128</td><td style=\"text-align: right;\">0.00463597 </td><td style=\"text-align: right;\">     1</td><td style=\"text-align: right;\">        0.580763</td><td style=\"text-align: right;\">119.317 </td><td style=\"text-align: right;\">   91.1238</td></tr>\n",
       "<tr><td>train_mlp_eb0f3_00009</td><td>TERMINATED</td><td>10.234.204.79:3157852</td><td style=\"text-align: right;\">        1024</td><td style=\"text-align: right;\">0.0621485  </td><td style=\"text-align: right;\">   100</td><td style=\"text-align: right;\">       31.9137  </td><td style=\"text-align: right;\"> 15.9689</td><td style=\"text-align: right;\">   86.4812</td></tr>\n",
       "<tr><td>train_mlp_eb0f3_00010</td><td>TERMINATED</td><td>10.234.204.79:3157852</td><td style=\"text-align: right;\">         128</td><td style=\"text-align: right;\">0.010009   </td><td style=\"text-align: right;\">     1</td><td style=\"text-align: right;\">        0.602533</td><td style=\"text-align: right;\">122.615 </td><td style=\"text-align: right;\">   86.88  </td></tr>\n",
       "<tr><td>train_mlp_eb0f3_00011</td><td>TERMINATED</td><td>10.234.204.79:3157852</td><td style=\"text-align: right;\">        1024</td><td style=\"text-align: right;\">0.000551726</td><td style=\"text-align: right;\">   100</td><td style=\"text-align: right;\">       29.2209  </td><td style=\"text-align: right;\"> 15.3524</td><td style=\"text-align: right;\">   92.535 </td></tr>\n",
       "<tr><td>train_mlp_eb0f3_00012</td><td>TERMINATED</td><td>10.234.204.79:3157852</td><td style=\"text-align: right;\">         512</td><td style=\"text-align: right;\">0.000101218</td><td style=\"text-align: right;\">     1</td><td style=\"text-align: right;\">        0.393358</td><td style=\"text-align: right;\"> 44.4644</td><td style=\"text-align: right;\">   70.2117</td></tr>\n",
       "<tr><td>train_mlp_eb0f3_00013</td><td>TERMINATED</td><td>10.234.204.79:3157852</td><td style=\"text-align: right;\">         512</td><td style=\"text-align: right;\">0.00308178 </td><td style=\"text-align: right;\">     4</td><td style=\"text-align: right;\">        1.12512 </td><td style=\"text-align: right;\"> 30.8914</td><td style=\"text-align: right;\">   91.809 </td></tr>\n",
       "<tr><td>train_mlp_eb0f3_00014</td><td>TERMINATED</td><td>10.234.204.79:3157852</td><td style=\"text-align: right;\">         256</td><td style=\"text-align: right;\">0.00347563 </td><td style=\"text-align: right;\">     1</td><td style=\"text-align: right;\">        0.464293</td><td style=\"text-align: right;\"> 60.4465</td><td style=\"text-align: right;\">   91.2056</td></tr>\n",
       "<tr><td>train_mlp_eb0f3_00015</td><td>TERMINATED</td><td>10.234.204.79:3157852</td><td style=\"text-align: right;\">        1024</td><td style=\"text-align: right;\">0.0387326  </td><td style=\"text-align: right;\">   100</td><td style=\"text-align: right;\">       29.5708  </td><td style=\"text-align: right;\"> 15.4324</td><td style=\"text-align: right;\">   91.7476</td></tr>\n",
       "<tr><td>train_mlp_eb0f3_00016</td><td>TERMINATED</td><td>10.234.204.79:3157852</td><td style=\"text-align: right;\">         128</td><td style=\"text-align: right;\">0.00040387 </td><td style=\"text-align: right;\">     1</td><td style=\"text-align: right;\">        0.653244</td><td style=\"text-align: right;\">119.388 </td><td style=\"text-align: right;\">   91.5227</td></tr>\n",
       "<tr><td>train_mlp_eb0f3_00017</td><td>TERMINATED</td><td>10.234.204.79:3157852</td><td style=\"text-align: right;\">          64</td><td style=\"text-align: right;\">0.000127475</td><td style=\"text-align: right;\">     1</td><td style=\"text-align: right;\">        1.06837 </td><td style=\"text-align: right;\">318.266 </td><td style=\"text-align: right;\">   70.4162</td></tr>\n",
       "<tr><td>train_mlp_eb0f3_00018</td><td>TERMINATED</td><td>10.234.204.79:3157852</td><td style=\"text-align: right;\">         128</td><td style=\"text-align: right;\">5.56458e-05</td><td style=\"text-align: right;\">     1</td><td style=\"text-align: right;\">        0.567038</td><td style=\"text-align: right;\">176.871 </td><td style=\"text-align: right;\">   12.6496</td></tr>\n",
       "<tr><td>train_mlp_eb0f3_00019</td><td>TERMINATED</td><td>10.234.204.79:3157852</td><td style=\"text-align: right;\">          64</td><td style=\"text-align: right;\">0.0002628  </td><td style=\"text-align: right;\">     1</td><td style=\"text-align: right;\">        0.910665</td><td style=\"text-align: right;\">237.122 </td><td style=\"text-align: right;\">   91.5227</td></tr>\n",
       "</tbody>\n",
       "</table>\n",
       "  </div>\n",
       "</div>\n",
       "<style>\n",
       ".tuneStatus {\n",
       "  color: var(--jp-ui-font-color1);\n",
       "}\n",
       ".tuneStatus .systemInfo {\n",
       "  display: flex;\n",
       "  flex-direction: column;\n",
       "}\n",
       ".tuneStatus td {\n",
       "  white-space: nowrap;\n",
       "}\n",
       ".tuneStatus .trialStatus {\n",
       "  display: flex;\n",
       "  flex-direction: column;\n",
       "}\n",
       ".tuneStatus h3 {\n",
       "  font-weight: bold;\n",
       "}\n",
       ".tuneStatus .hDivider {\n",
       "  border-bottom-width: var(--jp-border-width);\n",
       "  border-bottom-color: var(--jp-border-color0);\n",
       "  border-bottom-style: solid;\n",
       "}\n",
       ".tuneStatus .vDivider {\n",
       "  border-left-width: var(--jp-border-width);\n",
       "  border-left-color: var(--jp-border-color0);\n",
       "  border-left-style: solid;\n",
       "  margin: 0.5em 1em 0.5em 1em;\n",
       "}\n",
       "</style>\n"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2024-03-04 19:53:52,747\tWARNING worker.py:2065 -- Warning: The actor ImplicitFunc is very large (16 MiB). Check that its definition is not implicitly capturing a large array or other object in scope. Tip: use ray.put() to put large objects in the Ray object store.\n",
      "2024-03-04 19:57:50,042\tINFO tune.py:1154 -- Total run time: 241.13 seconds (240.81 seconds for the tuning loop).\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Best trial config: {'l1': 16, 'l2': 64, 'l3': 128, 'lr': 0.0005517260681532147, 'batch_size': 1024}\n",
      "Best trial final validation loss: 15.352375507354736\n",
      "Best trial final validation accuracy: 92.53502403108702\n",
      "> Number of parameters 11818\n",
      "Best trial test set accuracy: 89.8067287043665%\n"
     ]
    }
   ],
   "source": [
    "def main(num_samples=10, max_num_epochs=100, gpus_per_trial=1):\n",
    "    config = {\n",
    "        \"l1\": tune.sample_from(lambda _: 2 ** np.random.randint(3, 10)),\n",
    "        \"l2\": tune.sample_from(lambda _: 2 ** np.random.randint(3, 10)),\n",
    "        \"l3\": tune.sample_from(lambda _: 2 ** np.random.randint(3, 10)),\n",
    "        \"lr\": tune.loguniform(1e-5, 1e-1),\n",
    "        \"batch_size\": tune.choice([64, 128, 256, 512, 1024]),\n",
    "    }\n",
    "    scheduler = ASHAScheduler(\n",
    "        max_t=max_num_epochs,\n",
    "        grace_period=1,\n",
    "        reduction_factor=2)\n",
    "    \n",
    "    ray.shutdown()\n",
    "    ray.init(log_to_driver=False)\n",
    "    \n",
    "    tuner = tune.Tuner(\n",
    "        tune.with_resources(\n",
    "            tune.with_parameters(train_mlp),\n",
    "            resources={\"cpu\": 64, \"gpu\": gpus_per_trial}\n",
    "        ),\n",
    "        tune_config=tune.TuneConfig(\n",
    "            metric=\"loss\",\n",
    "            mode=\"min\",\n",
    "            scheduler=scheduler,\n",
    "            num_samples=num_samples,\n",
    "        ),\n",
    "        param_space=config,\n",
    "        run_config=air.RunConfig(\n",
    "            local_dir=local_dir,\n",
    "            name=\"mlp\",\n",
    "        )\n",
    "    )\n",
    "    results = tuner.fit()\n",
    "    \n",
    "    best_result = results.get_best_result(\"loss\", \"min\")\n",
    "\n",
    "    print(\"Best trial config: {}\".format(best_result.config))\n",
    "    print(\"Best trial final validation loss: {}\".format(\n",
    "        best_result.metrics[\"loss\"]))\n",
    "    print(\"Best trial final validation accuracy: {}\".format(\n",
    "        best_result.metrics[\"accuracy\"]))\n",
    "\n",
    "    test_best_model(best_result)\n",
    "    \n",
    "main(num_samples=20, max_num_epochs=100, gpus_per_trial=1)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "mykernel",
   "language": "python",
   "name": "mykernel"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.10"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}