TrainingModel.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "85d5a39c",
   "metadata": {},
   "outputs": [],
   "source": [
    "import sagemaker\n",
    "role = sagemaker.get_execution_role()\n",
    "#role = \"sagemaker-role\"\n",
    "# vpc_config_override = {'Subnets' : ['subnet-42856d3a'], 'SecurityGroupIds' :['sg-a8e070f0']}\n",
    "subnets = ['subnet-42856d3a']\n",
    "security_group_ids = ['sg-a8e070f0']\n",
    "#sg-a8e070f0"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "7272916d",
   "metadata": {},
   "outputs": [],
   "source": [
    "from sagemaker.pytorch import PyTorch\n",
    "estimator = PyTorch(\n",
    "    entry_point=\"src/main.py\",\n",
    "    role=role,\n",
    "    framework_version=\"1.5.0\",\n",
    "    py_version=\"py3\",\n",
    "    source_dir=\"/home/ec2-user/SageMaker/smoke-detection-smokeynet/\",\n",
    "    instance_count=1,\n",
    "    instance_type= \"ml.g4dn.xlarge\",  # \"ml.m5.xlarge\", # \n",
    "    hyperparameters={ \"experiment-name\" : \"Final_ResNet_Transformer_SpatialViT\" ,\n",
    "    #\"model-type-list\": [\"RawToTile_ResNet\" ,\"TileToTile_LSTM\" ,\"TileToTileImage_SpatialViT\"],\n",
    "    \"omit-list\": \"omit_no_xml\" ,\n",
    "    \"batch-size\": 2 ,\n",
    "    \"series-length\" : 2 ,\n",
    "    \"accumulate-grad-batches\" : 16 ,\n",
    "    \"num-workers\" : 4 ,\n",
    "    \"train-split-path\": './data/final_split/train_images_final.txt' ,\n",
    "    \"val-split-path\": './data/final_split/val_images_final.txt' ,\n",
    "    \"test-split-path\" : './data/final_split/test_images_final.txt' ,\n",
    "    \"tile-embedding-size\": 1000},\n",
    "    subnets=subnets,\n",
    "    security_group_ids=security_group_ids\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "98a2aa68",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2022-12-04 19:18:33 Starting - Starting the training job...\n",
      "2022-12-04 19:18:56 Starting - Preparing the instances for trainingProfilerReport-1670181512: InProgress\n",
      "......\n",
      "2022-12-04 19:19:59 Downloading - Downloading input data\n",
      "2022-12-04 19:19:59 Training - Downloading the training image............\n",
      "2022-12-04 19:22:00 Training - Training image download completed. Training in progress..\u001b[34mbash: cannot set terminal process group (-1): Inappropriate ioctl for device\u001b[0m\n",
      "\u001b[34mbash: no job control in this shell\u001b[0m\n",
      "\u001b[34m2022-12-04 19:22:02,611 sagemaker-training-toolkit INFO     Imported framework sagemaker_pytorch_container.training\u001b[0m\n",
      "\u001b[34m2022-12-04 19:22:02,646 sagemaker_pytorch_container.training INFO     Block until all host DNS lookups succeed.\u001b[0m\n",
      "\u001b[34m2022-12-04 19:22:02,654 sagemaker_pytorch_container.training INFO     Invoking user training script.\u001b[0m\n",
      "\u001b[34m2022-12-04 19:22:14,698 sagemaker-training-toolkit INFO     Installing dependencies from requirements.txt:\u001b[0m\n",
      "\u001b[34m/opt/conda/bin/python3.6 -m pip install -r requirements.txt\u001b[0m\n",
      "\u001b[34mCollecting transformers\n",
      "  Downloading transformers-4.18.0-py3-none-any.whl (4.0 MB)\u001b[0m\n",
      "\u001b[34mCollecting jupyterlab\u001b[0m\n",
      "\u001b[34m  Downloading jupyterlab-3.2.9-py3-none-any.whl (8.5 MB)\u001b[0m\n",
      "\u001b[34mRequirement already satisfied: numpy in /opt/conda/lib/python3.6/site-packages (from -r requirements.txt (line 3)) (1.16.4)\u001b[0m\n",
      "\u001b[34mRequirement already satisfied: matplotlib in /opt/conda/lib/python3.6/site-packages (from -r requirements.txt (line 4)) (3.3.4)\u001b[0m\n",
      "\u001b[34mRequirement already satisfied: torch in /opt/conda/lib/python3.6/site-packages (from -r requirements.txt (line 5)) (1.5.0)\u001b[0m\n",
      "\u001b[34mRequirement already satisfied: torchvision in /opt/conda/lib/python3.6/site-packages (from -r requirements.txt (line 6)) (0.6.0)\u001b[0m\n",
      "\u001b[34mCollecting pytorch-lightning\n",
      "  Downloading pytorch_lightning-1.5.10-py3-none-any.whl (527 kB)\u001b[0m\n",
      "\u001b[34mCollecting torchmetrics\n",
      "  Downloading torchmetrics-0.8.2-py3-none-any.whl (409 kB)\u001b[0m\n",
      "\u001b[34mCollecting sklearn\n",
      "  Downloading sklearn-0.0.post1.tar.gz (3.6 kB)\u001b[0m\n",
      "\u001b[34mCollecting opencv-python\n",
      "  Downloading opencv_python-4.6.0.66-cp36-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (60.9 MB)\u001b[0m\n",
      "\u001b[34mCollecting efficientnet-pytorch\n",
      "  Downloading efficientnet_pytorch-0.7.1.tar.gz (21 kB)\u001b[0m\n",
      "\u001b[34mRequirement already satisfied: ipython in /opt/conda/lib/python3.6/site-packages (from jupyterlab->-r requirements.txt (line 2)) (7.10.1)\u001b[0m\n",
      "\u001b[34mCollecting tornado>=6.1.0\n",
      "  Downloading tornado-6.1-cp36-cp36m-manylinux2010_x86_64.whl (427 kB)\u001b[0m\n",
      "\u001b[34mCollecting nbclassic~=0.2\n",
      "  Downloading nbclassic-0.3.5-py3-none-any.whl (25 kB)\u001b[0m\n",
      "\u001b[34mCollecting jupyter-server~=1.4\n",
      "  Downloading jupyter_server-1.13.1-py3-none-any.whl (396 kB)\u001b[0m\n",
      "\u001b[34mRequirement already satisfied: packaging in /opt/conda/lib/python3.6/site-packages (from jupyterlab->-r requirements.txt (line 2)) (20.9)\u001b[0m\n",
      "\u001b[34mCollecting jupyterlab-server~=2.3\n",
      "  Downloading jupyterlab_server-2.10.3-py3-none-any.whl (61 kB)\u001b[0m\n",
      "\u001b[34mCollecting jupyter-core\n",
      "  Downloading jupyter_core-4.9.2-py3-none-any.whl (86 kB)\u001b[0m\n",
      "\u001b[34mRequirement already satisfied: jinja2>=2.1 in /opt/conda/lib/python3.6/site-packages (from jupyterlab->-r requirements.txt (line 2)) (2.11.3)\u001b[0m\n",
      "\u001b[34mRequirement already satisfied: MarkupSafe>=0.23 in /opt/conda/lib/python3.6/site-packages (from jinja2>=2.1->jupyterlab->-r requirements.txt (line 2)) (1.1.1)\u001b[0m\n",
      "\u001b[34mCollecting terminado>=0.8.3\n",
      "  Downloading terminado-0.12.1-py3-none-any.whl (15 kB)\u001b[0m\n",
      "\u001b[34mCollecting pyzmq>=17\n",
      "  Downloading pyzmq-24.0.1-cp36-cp36m-manylinux_2_5_x86_64.manylinux1_x86_64.whl (1.1 MB)\u001b[0m\n",
      "\u001b[34mCollecting jupyter-client>=6.1.1\n",
      "  Downloading jupyter_client-7.1.2-py3-none-any.whl (130 kB)\u001b[0m\n",
      "\u001b[34mCollecting argon2-cffi\n",
      "  Downloading argon2_cffi-21.3.0-py3-none-any.whl (14 kB)\u001b[0m\n",
      "\u001b[34mCollecting nbformat\n",
      "  Downloading nbformat-5.1.3-py3-none-any.whl (178 kB)\u001b[0m\n",
      "\u001b[34mCollecting Send2Trash\n",
      "  Downloading Send2Trash-1.8.0-py3-none-any.whl (18 kB)\u001b[0m\n",
      "\u001b[34mRequirement already satisfied: ipython-genutils in /opt/conda/lib/python3.6/site-packages (from jupyter-server~=1.4->jupyterlab->-r requirements.txt (line 2)) (0.2.0)\u001b[0m\n",
      "\u001b[34mCollecting nbconvert\n",
      "  Downloading nbconvert-6.0.7-py3-none-any.whl (552 kB)\u001b[0m\n",
      "\u001b[34mCollecting prometheus-client\n",
      "  Downloading prometheus_client-0.15.0-py3-none-any.whl (60 kB)\u001b[0m\n",
      "\u001b[34mCollecting anyio<4,>=3.1.0\n",
      "  Downloading anyio-3.6.2-py3-none-any.whl (80 kB)\u001b[0m\n",
      "\u001b[34mCollecting websocket-client\n",
      "  Downloading websocket_client-1.3.1-py3-none-any.whl (54 kB)\u001b[0m\n",
      "\u001b[34mRequirement already satisfied: traitlets>=4.2.1 in /opt/conda/lib/python3.6/site-packages (from jupyter-server~=1.4->jupyterlab->-r requirements.txt (line 2)) (4.3.3)\u001b[0m\n",
      "\u001b[34mRequirement already satisfied: contextvars in /opt/conda/lib/python3.6/site-packages (from anyio<4,>=3.1.0->jupyter-server~=1.4->jupyterlab->-r requirements.txt (line 2)) (2.4)\u001b[0m\n",
      "\u001b[34mCollecting sniffio>=1.1\n",
      "  Downloading sniffio-1.2.0-py3-none-any.whl (10 kB)\u001b[0m\n",
      "\u001b[34mRequirement already satisfied: typing-extensions in /opt/conda/lib/python3.6/site-packages (from anyio<4,>=3.1.0->jupyter-server~=1.4->jupyterlab->-r requirements.txt (line 2)) (3.7.4.3)\u001b[0m\n",
      "\u001b[34mRequirement already satisfied: dataclasses in /opt/conda/lib/python3.6/site-packages (from anyio<4,>=3.1.0->jupyter-server~=1.4->jupyterlab->-r requirements.txt (line 2)) (0.8)\u001b[0m\n",
      "\u001b[34mRequirement already satisfied: idna>=2.8 in /opt/conda/lib/python3.6/site-packages (from anyio<4,>=3.1.0->jupyter-server~=1.4->jupyterlab->-r requirements.txt (line 2)) (2.8)\u001b[0m\n",
      "\u001b[34mCollecting entrypoints\n",
      "  Downloading entrypoints-0.4-py3-none-any.whl (5.3 kB)\u001b[0m\n",
      "\u001b[34mRequirement already satisfied: python-dateutil>=2.1 in /opt/conda/lib/python3.6/site-packages (from jupyter-client>=6.1.1->jupyter-server~=1.4->jupyterlab->-r requirements.txt (line 2)) (2.8.1)\u001b[0m\n",
      "\u001b[34mCollecting nest-asyncio>=1.5\n",
      "  Downloading nest_asyncio-1.5.6-py3-none-any.whl (5.2 kB)\u001b[0m\n",
      "\u001b[34mCollecting jsonschema>=3.0.1\n",
      "  Downloading jsonschema-3.2.0-py2.py3-none-any.whl (56 kB)\u001b[0m\n",
      "\u001b[34mCollecting json5\n",
      "  Downloading json5-0.9.10-py2.py3-none-any.whl (19 kB)\u001b[0m\n",
      "\u001b[34mRequirement already satisfied: requests in /opt/conda/lib/python3.6/site-packages (from jupyterlab-server~=2.3->jupyterlab->-r requirements.txt (line 2)) (2.22.0)\u001b[0m\n",
      "\u001b[34mCollecting babel\n",
      "  Downloading Babel-2.11.0-py3-none-any.whl (9.5 MB)\u001b[0m\n",
      "\u001b[34mRequirement already satisfied: setuptools in /opt/conda/lib/python3.6/site-packages (from jsonschema>=3.0.1->jupyterlab-server~=2.3->jupyterlab->-r requirements.txt (line 2)) (49.6.0.post20210108)\u001b[0m\n",
      "\u001b[34mRequirement already satisfied: importlib-metadata in /opt/conda/lib/python3.6/site-packages (from jsonschema>=3.0.1->jupyterlab-server~=2.3->jupyterlab->-r requirements.txt (line 2)) (3.7.2)\u001b[0m\n",
      "\u001b[34mCollecting pyrsistent>=0.14.0\n",
      "  Downloading pyrsistent-0.18.0-cp36-cp36m-manylinux1_x86_64.whl (117 kB)\u001b[0m\n",
      "\u001b[34mCollecting attrs>=17.4.0\n",
      "  Downloading attrs-22.1.0-py2.py3-none-any.whl (58 kB)\u001b[0m\n",
      "\u001b[34mRequirement already satisfied: six>=1.11.0 in /opt/conda/lib/python3.6/site-packages (from jsonschema>=3.0.1->jupyterlab-server~=2.3->jupyterlab->-r requirements.txt (line 2)) (1.15.0)\u001b[0m\n",
      "\u001b[34mCollecting notebook<7\n",
      "  Downloading notebook-6.4.10-py3-none-any.whl (9.9 MB)\u001b[0m\n",
      "\u001b[34mCollecting ipykernel\n",
      "  Downloading ipykernel-5.5.6-py3-none-any.whl (121 kB)\u001b[0m\n",
      "\u001b[34mCollecting nbclient<0.6.0,>=0.5.0\n",
      "  Downloading nbclient-0.5.9-py3-none-any.whl (69 kB)\u001b[0m\n",
      "\u001b[34mCollecting jupyterlab-pygments\n",
      "  Downloading jupyterlab_pygments-0.1.2-py2.py3-none-any.whl (4.6 kB)\u001b[0m\n",
      "\u001b[34mCollecting pandocfilters>=1.4.1\n",
      "  Downloading pandocfilters-1.5.0-py2.py3-none-any.whl (8.7 kB)\u001b[0m\n",
      "\u001b[34mCollecting testpath\n",
      "  Downloading testpath-0.6.0-py3-none-any.whl (83 kB)\u001b[0m\n",
      "\u001b[34mCollecting mistune<2,>=0.8.1\n",
      "  Downloading mistune-0.8.4-py2.py3-none-any.whl (16 kB)\u001b[0m\n",
      "\u001b[34mCollecting defusedxml\n",
      "  Downloading defusedxml-0.7.1-py2.py3-none-any.whl (25 kB)\u001b[0m\n",
      "\u001b[34mRequirement already satisfied: pygments>=2.4.1 in /opt/conda/lib/python3.6/site-packages (from nbconvert->jupyter-server~=1.4->jupyterlab->-r requirements.txt (line 2)) (2.7.1)\u001b[0m\n",
      "\u001b[34mCollecting bleach\n",
      "  Downloading bleach-4.1.0-py2.py3-none-any.whl (157 kB)\u001b[0m\n",
      "\u001b[34mCollecting async-generator\n",
      "  Downloading async_generator-1.10-py3-none-any.whl (18 kB)\u001b[0m\n",
      "\u001b[34mRequirement already satisfied: immutables>=0.9 in /opt/conda/lib/python3.6/site-packages (from contextvars->anyio<4,>=3.1.0->jupyter-server~=1.4->jupyterlab->-r requirements.txt (line 2)) (0.15)\u001b[0m\n",
      "\u001b[34mRequirement already satisfied: ptyprocess in /opt/conda/lib/python3.6/site-packages (from terminado>=0.8.3->jupyter-server~=1.4->jupyterlab->-r requirements.txt (line 2)) (0.6.0)\u001b[0m\n",
      "\u001b[34mRequirement already satisfied: decorator in /opt/conda/lib/python3.6/site-packages (from traitlets>=4.2.1->jupyter-server~=1.4->jupyterlab->-r requirements.txt (line 2)) (4.4.2)\u001b[0m\n",
      "\u001b[34mRequirement already satisfied: pyparsing!=2.0.4,!=2.1.2,!=2.1.6,>=2.0.3 in /opt/conda/lib/python3.6/site-packages (from matplotlib->-r requirements.txt (line 4)) (2.4.7)\u001b[0m\n",
      "\u001b[34mRequirement already satisfied: cycler>=0.10 in /opt/conda/lib/python3.6/site-packages (from matplotlib->-r requirements.txt (line 4)) (0.10.0)\u001b[0m\n",
      "\u001b[34mRequirement already satisfied: pillow>=6.2.0 in /opt/conda/lib/python3.6/site-packages (from matplotlib->-r requirements.txt (line 4)) (7.1.0)\u001b[0m\n",
      "\u001b[34mRequirement already satisfied: kiwisolver>=1.0.1 in /opt/conda/lib/python3.6/site-packages (from matplotlib->-r requirements.txt (line 4)) (1.3.1)\u001b[0m\n",
      "\u001b[34mRequirement already satisfied: PyYAML>=5.1 in /opt/conda/lib/python3.6/site-packages (from pytorch-lightning->-r requirements.txt (line 7)) (5.4.1)\u001b[0m\n",
      "\u001b[34mCollecting fsspec[http]!=2021.06.0,>=2021.05.0\n",
      "  Downloading fsspec-2022.1.0-py3-none-any.whl (133 kB)\u001b[0m\n",
      "\u001b[34mCollecting setuptools\n",
      "  Downloading setuptools-59.5.0-py3-none-any.whl (952 kB)\u001b[0m\n",
      "\u001b[34mCollecting tensorboard>=2.2.0\n",
      "  Downloading tensorboard-2.10.1-py3-none-any.whl (5.9 MB)\u001b[0m\n",
      "\u001b[34mCollecting pyDeprecate==0.3.1\n",
      "  Downloading pyDeprecate-0.3.1-py3-none-any.whl (10 kB)\u001b[0m\n",
      "\u001b[34mRequirement already satisfied: future>=0.17.1 in /opt/conda/lib/python3.6/site-packages (from pytorch-lightning->-r requirements.txt (line 7)) (0.17.1)\u001b[0m\n",
      "\u001b[34mRequirement already satisfied: tqdm>=4.41.0 in /opt/conda/lib/python3.6/site-packages (from pytorch-lightning->-r requirements.txt (line 7)) (4.56.0)\u001b[0m\n",
      "\u001b[34mCollecting numpy\n",
      "  Downloading numpy-1.19.5-cp36-cp36m-manylinux2010_x86_64.whl (14.8 MB)\u001b[0m\n",
      "\u001b[34mCollecting torch\n",
      "  Downloading torch-1.10.2-cp36-cp36m-manylinux1_x86_64.whl (881.9 MB)\u001b[0m\n",
      "\u001b[34mCollecting aiohttp\n",
      "  Downloading aiohttp-3.8.3-cp36-cp36m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (945 kB)\u001b[0m\n",
      "\u001b[34mCollecting grpcio>=1.24.3\n",
      "  Downloading grpcio-1.48.2-cp36-cp36m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (4.6 MB)\u001b[0m\n",
      "\u001b[34mCollecting google-auth<3,>=1.6.3\n",
      "  Downloading google_auth-2.15.0-py2.py3-none-any.whl (177 kB)\u001b[0m\n",
      "\u001b[34mRequirement already satisfied: protobuf<3.20,>=3.9.2 in /opt/conda/lib/python3.6/site-packages (from tensorboard>=2.2.0->pytorch-lightning->-r requirements.txt (line 7)) (3.15.6)\u001b[0m\n",
      "\u001b[34mRequirement already satisfied: wheel>=0.26 in /opt/conda/lib/python3.6/site-packages (from tensorboard>=2.2.0->pytorch-lightning->-r requirements.txt (line 7)) (0.36.2)\u001b[0m\n",
      "\u001b[34mRequirement already satisfied: werkzeug>=1.0.1 in /opt/conda/lib/python3.6/site-packages (from tensorboard>=2.2.0->pytorch-lightning->-r requirements.txt (line 7)) (1.0.1)\u001b[0m\n",
      "\u001b[34mCollecting markdown>=2.6.8\n",
      "  Downloading Markdown-3.3.7-py3-none-any.whl (97 kB)\u001b[0m\n",
      "\u001b[34mCollecting tensorboard-plugin-wit>=1.6.0\n",
      "  Downloading tensorboard_plugin_wit-1.8.1-py3-none-any.whl (781 kB)\u001b[0m\n",
      "\u001b[34mCollecting tensorboard-data-server<0.7.0,>=0.6.0\n",
      "  Downloading tensorboard_data_server-0.6.1-py3-none-manylinux2010_x86_64.whl (4.9 MB)\u001b[0m\n",
      "\u001b[34mCollecting absl-py>=0.4\n",
      "  Downloading absl_py-1.3.0-py3-none-any.whl (124 kB)\u001b[0m\n",
      "\u001b[34mCollecting google-auth-oauthlib<0.5,>=0.4.1\n",
      "  Downloading google_auth_oauthlib-0.4.6-py2.py3-none-any.whl (18 kB)\u001b[0m\n",
      "\u001b[34mCollecting cachetools<6.0,>=2.0.0\n",
      "  Downloading cachetools-4.2.4-py3-none-any.whl (10 kB)\u001b[0m\n",
      "\u001b[34mCollecting pyasn1-modules>=0.2.1\n",
      "  Downloading pyasn1_modules-0.2.8-py2.py3-none-any.whl (155 kB)\u001b[0m\n",
      "\u001b[34mRequirement already satisfied: rsa<5,>=3.1.4 in /opt/conda/lib/python3.6/site-packages (from google-auth<3,>=1.6.3->tensorboard>=2.2.0->pytorch-lightning->-r requirements.txt (line 7)) (4.7.2)\u001b[0m\n",
      "\u001b[34mCollecting requests-oauthlib>=0.7.0\n",
      "  Downloading requests_oauthlib-1.3.1-py2.py3-none-any.whl (23 kB)\u001b[0m\n",
      "\u001b[34mCollecting importlib-metadata\n",
      "  Downloading importlib_metadata-4.8.3-py3-none-any.whl (17 kB)\u001b[0m\n",
      "\u001b[34mRequirement already satisfied: zipp>=0.5 in /opt/conda/lib/python3.6/site-packages (from importlib-metadata->jsonschema>=3.0.1->jupyterlab-server~=2.3->jupyterlab->-r requirements.txt (line 2)) (3.4.1)\u001b[0m\n",
      "\u001b[34mRequirement already satisfied: pyasn1<0.5.0,>=0.4.6 in /opt/conda/lib/python3.6/site-packages (from pyasn1-modules>=0.2.1->google-auth<3,>=1.6.3->tensorboard>=2.2.0->pytorch-lightning->-r requirements.txt (line 7)) (0.4.8)\u001b[0m\n",
      "\u001b[34mRequirement already satisfied: urllib3!=1.25.0,!=1.25.1,<1.26,>=1.21.1 in /opt/conda/lib/python3.6/site-packages (from requests->jupyterlab-server~=2.3->jupyterlab->-r requirements.txt (line 2)) (1.25.11)\u001b[0m\n",
      "\u001b[34mRequirement already satisfied: certifi>=2017.4.17 in /opt/conda/lib/python3.6/site-packages (from requests->jupyterlab-server~=2.3->jupyterlab->-r requirements.txt (line 2)) (2020.12.5)\u001b[0m\n",
      "\u001b[34mRequirement already satisfied: chardet<3.1.0,>=3.0.2 in /opt/conda/lib/python3.6/site-packages (from requests->jupyterlab-server~=2.3->jupyterlab->-r requirements.txt (line 2)) (3.0.4)\u001b[0m\n",
      "\u001b[34mCollecting oauthlib>=3.0.0\n",
      "  Downloading oauthlib-3.2.2-py3-none-any.whl (151 kB)\u001b[0m\n",
      "\u001b[34mCollecting regex!=2019.12.17\n",
      "  Downloading regex-2022.10.31-cp36-cp36m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (756 kB)\u001b[0m\n",
      "\u001b[34mCollecting sacremoses\n",
      "  Downloading sacremoses-0.0.53.tar.gz (880 kB)\u001b[0m\n",
      "\u001b[34mCollecting tokenizers!=0.11.3,<0.13,>=0.11.1\n",
      "  Downloading tokenizers-0.12.1-cp36-cp36m-manylinux_2_12_x86_64.manylinux2010_x86_64.whl (6.6 MB)\u001b[0m\n",
      "\u001b[34mCollecting filelock\n",
      "  Downloading filelock-3.4.1-py3-none-any.whl (9.9 kB)\u001b[0m\n",
      "\u001b[34mCollecting huggingface-hub<1.0,>=0.1.0\n",
      "  Downloading huggingface_hub-0.4.0-py3-none-any.whl (67 kB)\u001b[0m\n",
      "\u001b[34mCollecting frozenlist>=1.1.1\n",
      "  Downloading frozenlist-1.2.0-cp36-cp36m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl (191 kB)\u001b[0m\n",
      "\u001b[34mCollecting aiosignal>=1.1.2\n",
      "  Downloading aiosignal-1.2.0-py3-none-any.whl (8.2 kB)\u001b[0m\n",
      "\u001b[34mCollecting charset-normalizer<3.0,>=2.0\n",
      "  Downloading charset_normalizer-2.1.1-py3-none-any.whl (39 kB)\u001b[0m\n",
      "\u001b[34mCollecting yarl<2.0,>=1.0\n",
      "  Downloading yarl-1.7.2-cp36-cp36m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl (270 kB)\u001b[0m\n",
      "\u001b[34mCollecting multidict<7.0,>=4.5\n",
      "  Downloading multidict-5.2.0-cp36-cp36m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl (159 kB)\u001b[0m\n",
      "\u001b[34mCollecting idna-ssl>=1.0\n",
      "  Downloading idna-ssl-1.1.0.tar.gz (3.4 kB)\u001b[0m\n",
      "\u001b[34mCollecting asynctest==0.13.0\n",
      "  Downloading asynctest-0.13.0-py3-none-any.whl (26 kB)\u001b[0m\n",
      "\u001b[34mCollecting async-timeout<5.0,>=4.0.0a3\n",
      "  Downloading async_timeout-4.0.2-py3-none-any.whl (5.8 kB)\u001b[0m\n",
      "\u001b[34mCollecting argon2-cffi-bindings\n",
      "  Downloading argon2_cffi_bindings-21.2.0-cp36-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (86 kB)\u001b[0m\n",
      "\u001b[34mRequirement already satisfied: cffi>=1.0.1 in /opt/conda/lib/python3.6/site-packages (from argon2-cffi-bindings->argon2-cffi->jupyter-server~=1.4->jupyterlab->-r requirements.txt (line 2)) (1.14.5)\u001b[0m\n",
      "\u001b[34mRequirement already satisfied: pycparser in /opt/conda/lib/python3.6/site-packages (from cffi>=1.0.1->argon2-cffi-bindings->argon2-cffi->jupyter-server~=1.4->jupyterlab->-r requirements.txt (line 2)) (2.20)\u001b[0m\n",
      "\u001b[34mRequirement already satisfied: pytz>=2015.7 in /opt/conda/lib/python3.6/site-packages (from babel->jupyterlab-server~=2.3->jupyterlab->-r requirements.txt (line 2)) (2021.1)\u001b[0m\n",
      "\u001b[34mCollecting webencodings\n",
      "  Downloading webencodings-0.5.1-py2.py3-none-any.whl (11 kB)\u001b[0m\n",
      "\u001b[34mRequirement already satisfied: backcall in /opt/conda/lib/python3.6/site-packages (from ipython->jupyterlab->-r requirements.txt (line 2)) (0.2.0)\u001b[0m\n",
      "\u001b[34mRequirement already satisfied: jedi>=0.10 in /opt/conda/lib/python3.6/site-packages (from ipython->jupyterlab->-r requirements.txt (line 2)) (0.17.2)\u001b[0m\n",
      "\u001b[34mRequirement already satisfied: pickleshare in /opt/conda/lib/python3.6/site-packages (from ipython->jupyterlab->-r requirements.txt (line 2)) (0.7.5)\u001b[0m\n",
      "\u001b[34mRequirement already satisfied: prompt-toolkit!=3.0.0,!=3.0.1,<3.1.0,>=2.0.0 in /opt/conda/lib/python3.6/site-packages (from ipython->jupyterlab->-r requirements.txt (line 2)) (3.0.8)\u001b[0m\n",
      "\u001b[34mRequirement already satisfied: pexpect in /opt/conda/lib/python3.6/site-packages (from ipython->jupyterlab->-r requirements.txt (line 2)) (4.8.0)\u001b[0m\n",
      "\u001b[34mRequirement already satisfied: parso<0.8.0,>=0.7.0 in /opt/conda/lib/python3.6/site-packages (from jedi>=0.10->ipython->jupyterlab->-r requirements.txt (line 2)) (0.7.0)\u001b[0m\n",
      "\u001b[34mRequirement already satisfied: wcwidth in /opt/conda/lib/python3.6/site-packages (from prompt-toolkit!=3.0.0,!=3.0.1,<3.1.0,>=2.0.0->ipython->jupyterlab->-r requirements.txt (line 2)) (0.2.5)\u001b[0m\n",
      "\u001b[34mRequirement already satisfied: click in /opt/conda/lib/python3.6/site-packages (from sacremoses->transformers->-r requirements.txt (line 1)) (7.1.2)\u001b[0m\n",
      "\u001b[34mRequirement already satisfied: joblib in /opt/conda/lib/python3.6/site-packages (from sacremoses->transformers->-r requirements.txt (line 1)) (1.0.1)\u001b[0m\n",
      "\u001b[34mBuilding wheels for collected packages: efficientnet-pytorch, sklearn, idna-ssl, sacremoses\n",
      "  Building wheel for efficientnet-pytorch (setup.py): started\u001b[0m\n",
      "\u001b[34m  Building wheel for efficientnet-pytorch (setup.py): finished with status 'done'\n",
      "  Created wheel for efficientnet-pytorch: filename=efficientnet_pytorch-0.7.1-py3-none-any.whl size=16446 sha256=466ed744307c616cd1d322aed64f88ca19b652b7aa52cbc5625b24fafb55073f\n",
      "  Stored in directory: /root/.cache/pip/wheels/63/17/7e/07f1c55a623c96ba9b291c5d2e4901afffb953b032c2470ca6\n",
      "  Building wheel for sklearn (setup.py): started\n",
      "  Building wheel for sklearn (setup.py): finished with status 'done'\n",
      "  Created wheel for sklearn: filename=sklearn-0.0.post1-py3-none-any.whl size=2343 sha256=66f4ad1cb28475ae98187524dd13e803816ebb5dea4a087eb0d649d6771f71a0\n",
      "  Stored in directory: /root/.cache/pip/wheels/a8/20/b1/0b08c6071caf725f6b3f48c30faec4a855bcae02df1e61ed6c\n",
      "  Building wheel for idna-ssl (setup.py): started\u001b[0m\n",
      "\u001b[34m  Building wheel for idna-ssl (setup.py): finished with status 'done'\n",
      "  Created wheel for idna-ssl: filename=idna_ssl-1.1.0-py3-none-any.whl size=3160 sha256=8147ca58ddf469154cc6d9b11fa91c9fb0a04ce3bbcf4103e283e026d163b4aa\n",
      "  Stored in directory: /root/.cache/pip/wheels/6a/f5/9c/f8331a854f7a8739cf0e74c13854e4dd7b1af11b04fe1dde13\n",
      "  Building wheel for sacremoses (setup.py): started\n",
      "  Building wheel for sacremoses (setup.py): finished with status 'done'\n",
      "  Created wheel for sacremoses: filename=sacremoses-0.0.53-py3-none-any.whl size=895254 sha256=0796aea7c35d6586f55652ef8df207a2075861d079f6eba2c78a4c8cf84eda68\n",
      "  Stored in directory: /root/.cache/pip/wheels/4c/64/31/e9900a234b23fb3e9dc565d6114a9d6ff84a72dbdd356502b4\u001b[0m\n",
      "\u001b[34mSuccessfully built efficientnet-pytorch sklearn idna-ssl sacremoses\u001b[0m\n",
      "\u001b[34mInstalling collected packages: setuptools, pyrsistent, importlib-metadata, attrs, tornado, pyzmq, nest-asyncio, jupyter-core, jsonschema, entrypoints, webencodings, nbformat, jupyter-client, async-generator, testpath, sniffio, pyasn1-modules, pandocfilters, oauthlib, nbclient, multidict, mistune, jupyterlab-pygments, frozenlist, defusedxml, cachetools, bleach, argon2-cffi-bindings, yarl, websocket-client, terminado, Send2Trash, requests-oauthlib, prometheus-client, nbconvert, ipykernel, idna-ssl, google-auth, charset-normalizer, asynctest, async-timeout, argon2-cffi, anyio, aiosignal, torch, tensorboard-plugin-wit, tensorboard-data-server, regex, pyDeprecate, numpy, notebook, markdown, jupyter-server, json5, grpcio, google-auth-oauthlib, fsspec, filelock, babel, aiohttp, absl-py, torchmetrics, tokenizers, tensorboard, sacremoses, nbclassic, jupyterlab-server, huggingface-hub, transformers, sklearn, pytorch-lightning, opencv-python, jupyterlab, efficientnet-pytorch\n",
      "  Attempting uninstall: setuptools\n",
      "    Found existing installation: setuptools 49.6.0.post20210108\n",
      "    Uninstalling setuptools-49.6.0.post20210108:\u001b[0m\n",
      "\u001b[34m      Successfully uninstalled setuptools-49.6.0.post20210108\n",
      "  Attempting uninstall: importlib-metadata\n",
      "    Found existing installation: importlib-metadata 3.7.2\n",
      "    Uninstalling importlib-metadata-3.7.2:\n",
      "      Successfully uninstalled importlib-metadata-3.7.2\u001b[0m\n",
      "\u001b[34m  Attempting uninstall: torch\n",
      "    Found existing installation: torch 1.5.0\n",
      "    Uninstalling torch-1.5.0:\n",
      "      Successfully uninstalled torch-1.5.0\u001b[0m\n",
      "\u001b[34m  Attempting uninstall: numpy\n",
      "    Found existing installation: numpy 1.16.4\n",
      "    Uninstalling numpy-1.16.4:\n",
      "      Successfully uninstalled numpy-1.16.4\u001b[0m\n",
      "\u001b[34mSuccessfully installed Send2Trash-1.8.0 absl-py-1.3.0 aiohttp-3.8.3 aiosignal-1.2.0 anyio-3.6.2 argon2-cffi-21.3.0 argon2-cffi-bindings-21.2.0 async-generator-1.10 async-timeout-4.0.2 asynctest-0.13.0 attrs-22.1.0 babel-2.11.0 bleach-4.1.0 cachetools-4.2.4 charset-normalizer-2.1.1 defusedxml-0.7.1 efficientnet-pytorch-0.7.1 entrypoints-0.4 filelock-3.4.1 frozenlist-1.2.0 fsspec-2022.1.0 google-auth-2.15.0 google-auth-oauthlib-0.4.6 grpcio-1.48.2 huggingface-hub-0.4.0 idna-ssl-1.1.0 importlib-metadata-4.8.3 ipykernel-5.5.6 json5-0.9.10 jsonschema-3.2.0 jupyter-client-7.1.2 jupyter-core-4.9.2 jupyter-server-1.13.1 jupyterlab-3.2.9 jupyterlab-pygments-0.1.2 jupyterlab-server-2.10.3 markdown-3.3.7 mistune-0.8.4 multidict-5.2.0 nbclassic-0.3.5 nbclient-0.5.9 nbconvert-6.0.7 nbformat-5.1.3 nest-asyncio-1.5.6 notebook-6.4.10 numpy-1.19.5 oauthlib-3.2.2 opencv-python-4.6.0.66 pandocfilters-1.5.0 prometheus-client-0.15.0 pyDeprecate-0.3.1 pyasn1-modules-0.2.8 pyrsistent-0.18.0 pytorch-lightning-1.5.10 pyzmq-24.0.1 regex-2022.10.31 requests-oauthlib-1.3.1 sacremoses-0.0.53 setuptools-59.5.0 sklearn-0.0.post1 sniffio-1.2.0 tensorboard-2.10.1 tensorboard-data-server-0.6.1 tensorboard-plugin-wit-1.8.1 terminado-0.12.1 testpath-0.6.0 tokenizers-0.12.1 torch-1.10.2 torchmetrics-0.8.2 tornado-6.1 transformers-4.18.0 webencodings-0.5.1 websocket-client-1.3.1 yarl-1.7.2\u001b[0m\n",
      "\u001b[34m2022-12-04 19:23:35,769 sagemaker-training-toolkit INFO     Invoking user script\u001b[0m\n",
      "\u001b[34mTraining Env:\u001b[0m\n",
      "\u001b[34m{\n",
      "    \"additional_framework_parameters\": {},\n",
      "    \"channel_input_dirs\": {\n",
      "        \"training\": \"/opt/ml/input/data/training\"\n",
      "    },\n",
      "    \"current_host\": \"algo-1\",\n",
      "    \"framework_module\": \"sagemaker_pytorch_container.training:main\",\n",
      "    \"hosts\": [\n",
      "        \"algo-1\"\n",
      "    ],\n",
      "    \"hyperparameters\": {\n",
      "        \"accumulate-grad-batches\": 16,\n",
      "        \"batch-size\": 2,\n",
      "        \"experiment-name\": \"Final_ResNet_Transformer_SpatialViT\",\n",
      "        \"num-workers\": 4,\n",
      "        \"omit-list\": \"omit_no_xml\",\n",
      "        \"series-length\": 2,\n",
      "        \"test-split-path\": \"./data/final_split/test_images_final.txt\",\n",
      "        \"tile-embedding-size\": 1000,\n",
      "        \"train-split-path\": \"./data/final_split/train_images_final.txt\",\n",
      "        \"val-split-path\": \"./data/final_split/val_images_final.txt\"\n",
      "    },\n",
      "    \"input_config_dir\": \"/opt/ml/input/config\",\n",
      "    \"input_data_config\": {\n",
      "        \"training\": {\n",
      "            \"TrainingInputMode\": \"File\",\n",
      "            \"S3DistributionType\": \"FullyReplicated\",\n",
      "            \"RecordWrapperType\": \"None\"\n",
      "        }\n",
      "    },\n",
      "    \"input_dir\": \"/opt/ml/input\",\n",
      "    \"is_master\": true,\n",
      "    \"job_name\": \"pytorch-training-2022-12-04-19-17-36-345\",\n",
      "    \"log_level\": 20,\n",
      "    \"master_hostname\": \"algo-1\",\n",
      "    \"model_dir\": \"/opt/ml/model\",\n",
      "    \"module_dir\": \"s3://sagemaker-us-west-2-113998783017/pytorch-training-2022-12-04-19-17-36-345/source/sourcedir.tar.gz\",\n",
      "    \"module_name\": \"src/main\",\n",
      "    \"network_interface_name\": \"eth0\",\n",
      "    \"num_cpus\": 4,\n",
      "    \"num_gpus\": 1,\n",
      "    \"output_data_dir\": \"/opt/ml/output/data\",\n",
      "    \"output_dir\": \"/opt/ml/output\",\n",
      "    \"output_intermediate_dir\": \"/opt/ml/output/intermediate\",\n",
      "    \"resource_config\": {\n",
      "        \"current_host\": \"algo-1\",\n",
      "        \"current_instance_type\": \"ml.g4dn.xlarge\",\n",
      "        \"current_group_name\": \"homogeneousCluster\",\n",
      "        \"hosts\": [\n",
      "            \"algo-1\"\n",
      "        ],\n",
      "        \"instance_groups\": [\n",
      "            {\n",
      "                \"instance_group_name\": \"homogeneousCluster\",\n",
      "                \"instance_type\": \"ml.g4dn.xlarge\",\n",
      "                \"hosts\": [\n",
      "                    \"algo-1\"\n",
      "                ]\n",
      "            }\n",
      "        ],\n",
      "        \"network_interface_name\": \"eth0\"\n",
      "    },\n",
      "    \"user_entry_point\": \"src/main.py\"\u001b[0m\n",
      "\u001b[34m}\u001b[0m\n",
      "\u001b[34mEnvironment variables:\u001b[0m\n",
      "\u001b[34mSM_HOSTS=[\"algo-1\"]\u001b[0m\n",
      "\u001b[34mSM_NETWORK_INTERFACE_NAME=eth0\u001b[0m\n",
      "\u001b[34mSM_HPS={\"accumulate-grad-batches\":16,\"batch-size\":2,\"experiment-name\":\"Final_ResNet_Transformer_SpatialViT\",\"num-workers\":4,\"omit-list\":\"omit_no_xml\",\"series-length\":2,\"test-split-path\":\"./data/final_split/test_images_final.txt\",\"tile-embedding-size\":1000,\"train-split-path\":\"./data/final_split/train_images_final.txt\",\"val-split-path\":\"./data/final_split/val_images_final.txt\"}\u001b[0m\n",
      "\u001b[34mSM_USER_ENTRY_POINT=src/main.py\u001b[0m\n",
      "\u001b[34mSM_FRAMEWORK_PARAMS={}\u001b[0m\n",
      "\u001b[34mSM_RESOURCE_CONFIG={\"current_group_name\":\"homogeneousCluster\",\"current_host\":\"algo-1\",\"current_instance_type\":\"ml.g4dn.xlarge\",\"hosts\":[\"algo-1\"],\"instance_groups\":[{\"hosts\":[\"algo-1\"],\"instance_group_name\":\"homogeneousCluster\",\"instance_type\":\"ml.g4dn.xlarge\"}],\"network_interface_name\":\"eth0\"}\u001b[0m\n",
      "\u001b[34mSM_INPUT_DATA_CONFIG={\"training\":{\"RecordWrapperType\":\"None\",\"S3DistributionType\":\"FullyReplicated\",\"TrainingInputMode\":\"File\"}}\u001b[0m\n",
      "\u001b[34mSM_OUTPUT_DATA_DIR=/opt/ml/output/data\u001b[0m\n",
      "\u001b[34mSM_CHANNELS=[\"training\"]\u001b[0m\n",
      "\u001b[34mSM_CURRENT_HOST=algo-1\u001b[0m\n",
      "\u001b[34mSM_MODULE_NAME=src/main\u001b[0m\n",
      "\u001b[34mSM_LOG_LEVEL=20\u001b[0m\n",
      "\u001b[34mSM_FRAMEWORK_MODULE=sagemaker_pytorch_container.training:main\u001b[0m\n",
      "\u001b[34mSM_INPUT_DIR=/opt/ml/input\u001b[0m\n",
      "\u001b[34mSM_INPUT_CONFIG_DIR=/opt/ml/input/config\u001b[0m\n",
      "\u001b[34mSM_OUTPUT_DIR=/opt/ml/output\u001b[0m\n",
      "\u001b[34mSM_NUM_CPUS=4\u001b[0m\n",
      "\u001b[34mSM_NUM_GPUS=1\u001b[0m\n",
      "\u001b[34mSM_MODEL_DIR=/opt/ml/model\u001b[0m\n",
      "\u001b[34mSM_MODULE_DIR=s3://sagemaker-us-west-2-113998783017/pytorch-training-2022-12-04-19-17-36-345/source/sourcedir.tar.gz\u001b[0m\n",
      "\u001b[34mSM_TRAINING_ENV={\"additional_framework_parameters\":{},\"channel_input_dirs\":{\"training\":\"/opt/ml/input/data/training\"},\"current_host\":\"algo-1\",\"framework_module\":\"sagemaker_pytorch_container.training:main\",\"hosts\":[\"algo-1\"],\"hyperparameters\":{\"accumulate-grad-batches\":16,\"batch-size\":2,\"experiment-name\":\"Final_ResNet_Transformer_SpatialViT\",\"num-workers\":4,\"omit-list\":\"omit_no_xml\",\"series-length\":2,\"test-split-path\":\"./data/final_split/test_images_final.txt\",\"tile-embedding-size\":1000,\"train-split-path\":\"./data/final_split/train_images_final.txt\",\"val-split-path\":\"./data/final_split/val_images_final.txt\"},\"input_config_dir\":\"/opt/ml/input/config\",\"input_data_config\":{\"training\":{\"RecordWrapperType\":\"None\",\"S3DistributionType\":\"FullyReplicated\",\"TrainingInputMode\":\"File\"}},\"input_dir\":\"/opt/ml/input\",\"is_master\":true,\"job_name\":\"pytorch-training-2022-12-04-19-17-36-345\",\"log_level\":20,\"master_hostname\":\"algo-1\",\"model_dir\":\"/opt/ml/model\",\"module_dir\":\"s3://sagemaker-us-west-2-113998783017/pytorch-training-2022-12-04-19-17-36-345/source/sourcedir.tar.gz\",\"module_name\":\"src/main\",\"network_interface_name\":\"eth0\",\"num_cpus\":4,\"num_gpus\":1,\"output_data_dir\":\"/opt/ml/output/data\",\"output_dir\":\"/opt/ml/output\",\"output_intermediate_dir\":\"/opt/ml/output/intermediate\",\"resource_config\":{\"current_group_name\":\"homogeneousCluster\",\"current_host\":\"algo-1\",\"current_instance_type\":\"ml.g4dn.xlarge\",\"hosts\":[\"algo-1\"],\"instance_groups\":[{\"hosts\":[\"algo-1\"],\"instance_group_name\":\"homogeneousCluster\",\"instance_type\":\"ml.g4dn.xlarge\"}],\"network_interface_name\":\"eth0\"},\"user_entry_point\":\"src/main.py\"}\u001b[0m\n",
      "\u001b[34mSM_USER_ARGS=[\"--accumulate-grad-batches\",\"16\",\"--batch-size\",\"2\",\"--experiment-name\",\"Final_ResNet_Transformer_SpatialViT\",\"--num-workers\",\"4\",\"--omit-list\",\"omit_no_xml\",\"--series-length\",\"2\",\"--test-split-path\",\"./data/final_split/test_images_final.txt\",\"--tile-embedding-size\",\"1000\",\"--train-split-path\",\"./data/final_split/train_images_final.txt\",\"--val-split-path\",\"./data/final_split/val_images_final.txt\"]\u001b[0m\n",
      "\u001b[34mSM_OUTPUT_INTERMEDIATE_DIR=/opt/ml/output/intermediate\u001b[0m\n",
      "\u001b[34mSM_CHANNEL_TRAINING=/opt/ml/input/data/training\u001b[0m\n",
      "\u001b[34mSM_HP_ACCUMULATE-GRAD-BATCHES=16\u001b[0m\n",
      "\u001b[34mSM_HP_BATCH-SIZE=2\u001b[0m\n",
      "\u001b[34mSM_HP_EXPERIMENT-NAME=Final_ResNet_Transformer_SpatialViT\u001b[0m\n",
      "\u001b[34mSM_HP_NUM-WORKERS=4\u001b[0m\n",
      "\u001b[34mSM_HP_OMIT-LIST=omit_no_xml\u001b[0m\n",
      "\u001b[34mSM_HP_SERIES-LENGTH=2\u001b[0m\n",
      "\u001b[34mSM_HP_TEST-SPLIT-PATH=./data/final_split/test_images_final.txt\u001b[0m\n",
      "\u001b[34mSM_HP_TILE-EMBEDDING-SIZE=1000\u001b[0m\n",
      "\u001b[34mSM_HP_TRAIN-SPLIT-PATH=./data/final_split/train_images_final.txt\u001b[0m\n",
      "\u001b[34mSM_HP_VAL-SPLIT-PATH=./data/final_split/val_images_final.txt\u001b[0m\n",
      "\u001b[34mPYTHONPATH=/opt/ml/code:/opt/conda/bin:/opt/conda/lib/python36.zip:/opt/conda/lib/python3.6:/opt/conda/lib/python3.6/lib-dynload:/opt/conda/lib/python3.6/site-packages\u001b[0m\n",
      "\u001b[34mInvoking script with the following command:\u001b[0m\n",
      "\u001b[34m/opt/conda/bin/python3.6 src/main.py --accumulate-grad-batches 16 --batch-size 2 --experiment-name Final_ResNet_Transformer_SpatialViT --num-workers 4 --omit-list omit_no_xml --series-length 2 --test-split-path ./data/final_split/test_images_final.txt --tile-embedding-size 1000 --train-split-path ./data/final_split/train_images_final.txt --val-split-path ./data/final_split/val_images_final.txt\u001b[0m\n",
      "\u001b[34mTrain data path:  /opt/ml/input/data/training\u001b[0m\n",
      "\u001b[34mtotal 26908\u001b[0m\n",
      "\u001b[34mdrwxrwxr-x 319 1000 1000    22528 Nov 18 17:03 HPWREN-FIgLib-Data\u001b[0m\n",
      "\u001b[34mdrwxr-xr-x 149 1000 1000    14336 Jun 22  2021 drive_clone_numpy\u001b[0m\n",
      "\u001b[34m-rw-rw-r--   1 1000 1000 27517678 Nov 19 17:39 drive_clone_numpy_nearest_contour.tar.gz\u001b[0m\n",
      "\u001b[34mList of the training dir 0\u001b[0m\n",
      "\u001b[34mExperiment:  Final_ResNet_Transformer_SpatialViT\u001b[0m\n",
      "\u001b[34m- IS_DEBUG:  False\u001b[0m\n",
      "\u001b[34m- IS_TEST_ONLY:  False\u001b[0m\n",
      "\u001b[34mInitializing MainModel...\u001b[0m\n",
      "\u001b[34m- RawToTile_ResNet_small\u001b[0m\n",
      "\u001b[34m- TileToTile_LSTM\u001b[0m\n",
      "\u001b[34m- TileToTileImage_SpatialViT\u001b[0m\n",
      "\u001b[34m-- Tile Loss: BCE Loss\u001b[0m\n",
      "\u001b[34mInitializing MainModel Complete.\u001b[0m\n",
      "\u001b[34mInitializing LightningModule...\u001b[0m\n",
      "\u001b[34mInitializing LightningModule Complete.\u001b[0m\n",
      "\u001b[34mSetting Up Data...\u001b[0m\n",
      "\u001b[34mSetting Up Data Complete.\u001b[0m\n",
      "\u001b[34mOptimizer: SGD\u001b[0m\n",
      "\u001b[34mLearning Rate:  0.001\u001b[0m\n",
      "\u001b[34m#015Validation sanity check: 0it [00:00, ?it/s]#015Validation sanity check:   0%|          | 0/2 [00:00<?, ?it/s]#015Validation sanity check:  50%|█████     | 1/2 [00:02<00:02,  2.58s/it]#015Validation sanity check: 100%|██████████| 2/2 [00:02<00:00,  1.18s/it]#015                                                                      #015#015Training: 0it [00:00, ?it/s]#015Training:   0%|          | 0/156 [00:00<?, ?it/s]#015Epoch 0:   0%|          | 0/156 [00:00<?, ?it/s] #015Epoch 0:   1%|          | 1/156 [00:04<12:07,  4.69s/it]#015Epoch 0:   1%|          | 1/156 [00:04<12:07,  4.69s/it, loss=25, v_num=0]#015Epoch 0:   1%|▏         | 2/156 [00:05<06:55,  2.70s/it, loss=25, v_num=0]#015Epoch 0:   1%|▏         | 2/156 [00:05<06:55,  2.70s/it, loss=12.5, v_num=0]#015Epoch 0:   2%|▏         | 3/156 [00:06<05:12,  2.05s/it, loss=12.5, v_num=0]#015Epoch 0:   2%|▏         | 3/156 [00:06<05:12,  2.05s/it, loss=20.5, v_num=0]#015Epoch 0:   3%|▎         | 4/156 [00:06<04:16,  1.69s/it, loss=20.5, v_num=0]#015Epoch 0:   3%|▎         | 4/156 [00:06<04:17,  1.70s/it, loss=21.7, v_num=0]#015Epoch 0:   3%|▎         | 5/156 [00:07<03:38,  1.45s/it, loss=21.7, v_num=0]#015Epoch 0:   3%|▎         | 5/156 [00:07<03:38,  1.45s/it, loss=22.3, v_num=0]#015Epoch 0:   4%|▍         | 6/156 [00:07<03:12,  1.29s/it, loss=22.3, v_num=0]#015Epoch 0:   4%|▍         | 6/156 [00:07<03:13,  1.29s/it, loss=22.8, v_num=0]#015Epoch 0:   4%|▍         | 7/156 [00:08<02:54,  1.17s/it, loss=22.8, v_num=0]#015Epoch 0:   4%|▍         | 7/156 [00:08<02:54,  1.17s/it, loss=23.1, v_num=0]#015Epoch 0:   5%|▌         | 8/156 [00:08<02:40,  1.09s/it, loss=23.1, v_num=0]#015Epoch 0:   5%|▌         | 8/156 [00:08<02:40,  1.09s/it, loss=23.3, v_num=0]#015Epoch 0:   6%|▌         | 9/156 [00:09<02:30,  1.02s/it, loss=23.3, v_num=0]#015Epoch 0:   6%|▌         | 9/156 [00:09<02:30,  1.02s/it, loss=23.5, v_num=0]#015Epoch 0:   6%|▋         | 10/156 [00:09<02:20,  1.04it/s, loss=23.5, v_num=0]#015Epoch 0:   6%|▋         | 10/156 [00:09<02:20,  1.04it/s, loss=23.7, v_num=0]#015Epoch 0:   7%|▋         | 11/156 [00:10<02:14,  1.08it/s, loss=23.7, v_num=0]#015Epoch 0:   7%|▋         | 11/156 [00:10<02:14,  1.08it/s, loss=23.8, v_num=0]#015Epoch 0:   8%|▊         | 12/156 [00:10<02:09,  1.11it/s, loss=23.8, v_num=0]#015Epoch 0:   8%|▊         | 12/156 [00:10<02:09,  1.11it/s, loss=23.9, v_num=0]#015Epoch 0:   8%|▊         | 13/156 [00:11<02:05,  1.14it/s, loss=23.9, v_num=0]#015Epoch 0:   8%|▊         | 13/156 [00:11<02:05,  1.14it/s, loss=24.9, v_num=0]#015Epoch 0:   9%|▉         | 14/156 [00:11<02:00,  1.18it/s, loss=24.9, v_num=0]#015Epoch 0:   9%|▉         | 14/156 [00:11<02:00,  1.18it/s, loss=25.7, v_num=0]#015Epoch 0:  10%|▉         | 15/156 [00:12<01:56,  1.21it/s, loss=25.7, v_num=0]#015Epoch 0:  10%|▉         | 15/156 [00:12<01:56,  1.21it/s, loss=26.4, v_num=0]#015Epoch 0:  10%|█         | 16/156 [00:12<01:53,  1.24it/s, loss=26.4, v_num=0]#015Epoch 0:  10%|█         | 16/156 [00:12<01:53,  1.24it/s, loss=27.1, v_num=0]#015Epoch 0:  11%|█         | 17/156 [00:13<01:50,  1.26it/s, loss=27.1, v_num=0]#015Epoch 0:  11%|█         | 17/156 [00:13<01:50,  1.26it/s, loss=26.9, v_num=0]#015Epoch 0:  12%|█▏        | 18/156 [00:14<01:47,  1.28it/s, loss=26.9, v_num=0]#015Epoch 0:  12%|█▏        | 18/156 [00:14<01:47,  1.28it/s, loss=26.8, v_num=0]#015Epoch 0:  12%|█▏        | 19/156 [00:14<01:44,  1.30it/s, loss=26.8, v_num=0]#015Epoch 0:  12%|█▏        | 19/156 [00:14<01:44,  1.30it/s, loss=27.3, v_num=0]#015Epoch 0:  13%|█▎        | 20/156 [00:15<01:42,  1.33it/s, loss=27.3, v_num=0]#015Epoch 0:  13%|█▎        | 20/156 [00:15<01:42,  1.33it/s, loss=27.8, v_num=0]#015Epoch 0:  13%|█▎        | 21/156 [00:15<01:39,  1.35it/s, loss=27.8, v_num=0]#015Epoch 0:  13%|█▎        | 21/156 [00:15<01:39,  1.35it/s, loss=27.8, v_num=0]#015Epoch 0:  14%|█▍        | 22/156 [00:16<01:37,  1.37it/s, loss=27.8, v_num=0]#015Epoch 0:  14%|█▍        | 22/156 [00:16<01:37,  1.37it/s, loss=29, v_num=0]  #015Epoch 0:  15%|█▍        | 23/156 [00:16<01:35,  1.39it/s, loss=29, v_num=0]#015Epoch 0:  15%|█▍        | 23/156 [00:16<01:35,  1.39it/s, loss=28.5, v_num=0]#015Epoch 0:  15%|█▌        | 24/156 [00:17<01:33,  1.41it/s, loss=28.5, v_num=0]#015Epoch 0:  15%|█▌        | 24/156 [00:17<01:33,  1.41it/s, loss=27.2, v_num=0]#015Epoch 0:  16%|█▌        | 25/156 [00:17<01:31,  1.43it/s, loss=27.2, v_num=0]#015Epoch 0:  16%|█▌        | 25/156 [00:17<01:31,  1.43it/s, loss=27.8, v_num=0]#015Epoch 0:  17%|█▋        | 26/156 [00:18<01:30,  1.44it/s, loss=27.8, v_num=0]#015Epoch 0:  17%|█▋        | 26/156 [00:18<01:30,  1.44it/s, loss=26.5, v_num=0]#015Epoch 0:  17%|█▋        | 27/156 [00:18<01:28,  1.46it/s, loss=26.5, v_num=0]#015Epoch 0:  17%|█▋        | 27/156 [00:18<01:28,  1.46it/s, loss=25.3, v_num=0]#015Epoch 0:  18%|█▊        | 28/156 [00:19<01:27,  1.47it/s, loss=25.3, v_num=0]#015Epoch 0:  18%|█▊        | 28/156 [00:19<01:27,  1.47it/s, loss=24.6, v_num=0]#015Epoch 0:  19%|█▊        | 29/156 [00:19<01:25,  1.49it/s, loss=24.6, v_num=0]#015Epoch 0:  19%|█▊        | 29/156 [00:19<01:25,  1.49it/s, loss=23.4, v_num=0]#015Epoch 0:  19%|█▉        | 30/156 [00:20<01:24,  1.50it/s, loss=23.4, v_num=0]#015Epoch 0:  19%|█▉        | 30/156 [00:20<01:24,  1.50it/s, loss=23.9, v_num=0]#015Epoch 0:  20%|█▉        | 31/156 [00:20<01:22,  1.51it/s, loss=23.9, v_num=0]#015Epoch 0:  20%|█▉        | 31/156 [00:20<01:22,  1.51it/s, loss=23.9, v_num=0]#015Epoch 0:  21%|██        | 32/156 [00:21<01:21,  1.52it/s, loss=23.9, v_num=0]#015Epoch 0:  21%|██        | 32/156 [00:21<01:21,  1.52it/s, loss=23.9, v_num=0]#015Epoch 0:  21%|██        | 33/156 [00:21<01:20,  1.53it/s, loss=23.9, v_num=0]#015Epoch 0:  21%|██        | 33/156 [00:21<01:20,  1.53it/s, loss=23.4, v_num=0]#015Epoch 0:  22%|██▏       | 34/156 [00:22<01:19,  1.54it/s, loss=23.4, v_num=0]#015Epoch 0:  22%|██▏       | 34/156 [00:22<01:19,  1.54it/s, loss=23.4, v_num=0]#015Epoch 0:  22%|██▏       | 35/156 [00:22<01:17,  1.55it/s, loss=23.4, v_num=0]#015Epoch 0:  22%|██▏       | 35/156 [00:22<01:17,  1.55it/s, loss=22.8, v_num=0]#015Epoch 0:  23%|██▎       | 36/156 [00:23<01:16,  1.56it/s, loss=22.8, v_num=0]#015Epoch 0:  23%|██▎       | 36/156 [00:23<01:16,  1.56it/s, loss=22.2, v_num=0]#015Epoch 0:  24%|██▎       | 37/156 [00:23<01:15,  1.57it/s, loss=22.2, v_num=0]#015Epoch 0:  24%|██▎       | 37/156 [00:23<01:15,  1.57it/s, loss=22.8, v_num=0]#015Epoch 0:  24%|██▍       | 38/156 [00:24<01:14,  1.58it/s, loss=22.8, v_num=0]#015Epoch 0:  24%|██▍       | 38/156 [00:24<01:14,  1.58it/s, loss=22.8, v_num=0]#015Epoch 0:  25%|██▌       | 39/156 [00:24<01:13,  1.58it/s, loss=22.8, v_num=0]#015Epoch 0:  25%|██▌       | 39/156 [00:24<01:13,  1.58it/s, loss=22.2, v_num=0]#015Epoch 0:  26%|██▌       | 40/156 [00:25<01:12,  1.59it/s, loss=22.2, v_num=0]#015Epoch 0:  26%|██▌       | 40/156 [00:25<01:12,  1.59it/s, loss=22.2, v_num=0]#015Epoch 0:  26%|██▋       | 41/156 [00:25<01:11,  1.60it/s, loss=22.2, v_num=0]#015Epoch 0:  26%|██▋       | 41/156 [00:25<01:11,  1.60it/s, loss=22.2, v_num=0]#015Epoch 0:  27%|██▋       | 42/156 [00:26<01:10,  1.61it/s, loss=22.2, v_num=0]#015Epoch 0:  27%|██▋       | 42/156 [00:26<01:10,  1.61it/s, loss=21, v_num=0]  #015Epoch 0:  28%|██▊       | 43/156 [00:26<01:09,  1.61it/s, loss=21, v_num=0]#015Epoch 0:  28%|██▊       | 43/156 [00:26<01:09,  1.61it/s, loss=21, v_num=0]#015Epoch 0:  28%|██▊       | 44/156 [00:27<01:08,  1.62it/s, loss=21, v_num=0]#015Epoch 0:  28%|██▊       | 44/156 [00:27<01:08,  1.62it/s, loss=22.8, v_num=0]#015Epoch 0:  29%|██▉       | 45/156 [00:27<01:08,  1.63it/s, loss=22.8, v_num=0]#015Epoch 0:  29%|██▉       | 45/156 [00:27<01:08,  1.63it/s, loss=21, v_num=0]  #015Epoch 0:  29%|██▉       | 46/156 [00:28<01:07,  1.64it/s, loss=21, v_num=0]#015Epoch 0:  29%|██▉       | 46/156 [00:28<01:07,  1.64it/s, loss=22.2, v_num=0]#015Epoch 0:  30%|███       | 47/156 [00:28<01:06,  1.64it/s, loss=22.2, v_num=0]#015Epoch 0:  30%|███       | 47/156 [00:28<01:06,  1.64it/s, loss=23.4, v_num=0]#015Epoch 0:  31%|███       | 48/156 [00:29<01:05,  1.64it/s, loss=23.4, v_num=0]#015Epoch 0:  31%|███       | 48/156 [00:29<01:05,  1.64it/s, loss=22.9, v_num=0]#015Epoch 0:  31%|███▏      | 49/156 [00:29<01:04,  1.65it/s, loss=22.9, v_num=0]#015Epoch 0:  31%|███▏      | 49/156 [00:29<01:04,  1.65it/s, loss=24.1, v_num=0]#015Epoch 0:  32%|███▏      | 50/156 [00:30<01:04,  1.65it/s, loss=24.1, v_num=0]#015Epoch 0:  32%|███▏      | 50/156 [00:30<01:04,  1.65it/s, loss=23.5, v_num=0]#015Epoch 0:  33%|███▎      | 51/156 [00:30<01:03,  1.66it/s, loss=23.5, v_num=0]#015Epoch 0:  33%|███▎      | 51/156 [00:30<01:03,  1.66it/s, loss=23.5, v_num=0]#015Epoch 0:  33%|███▎      | 52/156 [00:31<01:02,  1.67it/s, loss=23.5, v_num=0]#015Epoch 0:  33%|███▎      | 52/156 [00:31<01:02,  1.67it/s, loss=23.5, v_num=0]#015Epoch 0:  34%|███▍      | 53/156 [00:31<01:01,  1.67it/s, loss=23.5, v_num=0]#015Epoch 0:  34%|███▍      | 53/156 [00:31<01:01,  1.67it/s, loss=23.5, v_num=0]#015Epoch 0:  35%|███▍      | 54/156 [00:32<01:00,  1.68it/s, loss=23.5, v_num=0]#015Epoch 0:  35%|███▍      | 54/156 [00:32<01:00,  1.68it/s, loss=23, v_num=0]  #015Epoch 0:  35%|███▌      | 55/156 [00:32<00:59,  1.69it/s, loss=23, v_num=0]#015Epoch 0:  35%|███▌      | 55/156 [00:32<00:59,  1.69it/s, loss=21.7, v_num=0]#015Epoch 0:  36%|███▌      | 56/156 [00:33<00:59,  1.69it/s, loss=21.7, v_num=0]#015Epoch 0:  36%|███▌      | 56/156 [00:33<00:59,  1.69it/s, loss=21.7, v_num=0]#015Epoch 0:  37%|███▋      | 57/156 [00:33<00:58,  1.69it/s, loss=21.7, v_num=0]#015Epoch 0:  37%|███▋      | 57/156 [00:33<00:58,  1.69it/s, loss=21.1, v_num=0]#015Epoch 0:  37%|███▋      | 58/156 [00:34<00:57,  1.70it/s, loss=21.1, v_num=0]#015Epoch 0:  37%|███▋      | 58/156 [00:34<00:57,  1.70it/s, loss=19.9, v_num=0]#015Epoch 0:  38%|███▊      | 59/156 [00:34<00:56,  1.70it/s, loss=19.9, v_num=0]#015Epoch 0:  38%|███▊      | 59/156 [00:34<00:56,  1.70it/s, loss=19.9, v_num=0]#015Epoch 0:  38%|███▊      | 60/156 [00:35<00:56,  1.71it/s, loss=19.9, v_num=0]#015Epoch 0:  38%|███▊      | 60/156 [00:35<00:56,  1.71it/s, loss=19.9, v_num=0]#015Epoch 0:  39%|███▉      | 61/156 [00:35<00:55,  1.72it/s, loss=19.9, v_num=0]#015Epoch 0:  39%|███▉      | 61/156 [00:35<00:55,  1.72it/s, loss=18.6, v_num=0]#015Epoch 0:  40%|███▉      | 62/156 [00:36<00:54,  1.72it/s, loss=18.6, v_num=0]#015Epoch 0:  40%|███▉      | 62/156 [00:36<00:54,  1.72it/s, loss=19.9, v_num=0]#015Epoch 0:  40%|████      | 63/156 [00:36<00:53,  1.72it/s, loss=19.9, v_num=0]#015Epoch 0:  40%|████      | 63/156 [00:36<00:53,  1.72it/s, loss=18.6, v_num=0]#015Epoch 0:  41%|████      | 64/156 [00:37<00:53,  1.73it/s, loss=18.6, v_num=0]#015Epoch 0:  41%|████      | 64/156 [00:37<00:53,  1.73it/s, loss=18.1, v_num=0]#015Epoch 0:  42%|████▏     | 65/156 [00:37<00:52,  1.73it/s, loss=18.1, v_num=0]#015Epoch 0:  42%|████▏     | 65/156 [00:37<00:52,  1.73it/s, loss=18.1, v_num=0]#015Epoch 0:  42%|████▏     | 66/156 [00:38<00:51,  1.74it/s, loss=18.1, v_num=0]#015Epoch 0:  42%|████▏     | 66/156 [00:38<00:51,  1.74it/s, loss=16.8, v_num=0]#015Epoch 0:  43%|████▎     | 67/156 [00:38<00:51,  1.74it/s, loss=16.8, v_num=0]#015Epoch 0:  43%|████▎     | 67/156 [00:38<00:51,  1.74it/s, loss=17.4, v_num=0]#015Epoch 0:  44%|████▎     | 68/156 [00:38<00:50,  1.74it/s, loss=17.4, v_num=0]#015Epoch 0:  44%|████▎     | 68/156 [00:38<00:50,  1.74it/s, loss=18.6, v_num=0]#015Epoch 0:  44%|████▍     | 69/156 [00:39<00:49,  1.75it/s, loss=18.6, v_num=0]#015Epoch 0:  44%|████▍     | 69/156 [00:39<00:49,  1.75it/s, loss=18.6, v_num=0]#015Epoch 0:  45%|████▍     | 70/156 [00:39<00:49,  1.75it/s, loss=18.6, v_num=0]#015Epoch 0:  45%|████▍     | 70/156 [00:39<00:49,  1.75it/s, loss=18.6, v_num=0]#015Epoch 0:  46%|████▌     | 71/156 [00:40<00:48,  1.76it/s, loss=18.6, v_num=0]#015Epoch 0:  46%|████▌     | 71/156 [00:40<00:48,  1.76it/s, loss=19.2, v_num=0]#015Epoch 0:  46%|████▌     | 72/156 [00:40<00:47,  1.76it/s, loss=19.2, v_num=0]#015Epoch 0:  46%|████▌     | 72/156 [00:40<00:47,  1.76it/s, loss=19.2, v_num=0]#015Epoch 0:  47%|████▋     | 73/156 [00:41<00:46,  1.77it/s, loss=19.2, v_num=0]#015Epoch 0:  47%|████▋     | 73/156 [00:41<00:46,  1.77it/s, loss=19.7, v_num=0]#015Epoch 0:  47%|████▋     | 74/156 [00:41<00:46,  1.77it/s, loss=19.7, v_num=0]#015Epoch 0:  47%|████▋     | 74/156 [00:41<00:46,  1.77it/s, loss=20.3, v_num=0]#015Epoch 0:  48%|████▊     | 75/156 [00:42<00:45,  1.78it/s, loss=20.3, v_num=0]#015Epoch 0:  48%|████▊     | 75/156 [00:42<00:45,  1.78it/s, loss=21.6, v_num=0]#015Epoch 0:  49%|████▊     | 76/156 [00:42<00:44,  1.78it/s, loss=21.6, v_num=0]#015Epoch 0:  49%|████▊     | 76/156 [00:42<00:44,  1.78it/s, loss=20.3, v_num=0]#015Epoch 0:  49%|████▉     | 77/156 [00:43<00:44,  1.79it/s, loss=20.3, v_num=0]#015Epoch 0:  49%|████▉     | 77/156 [00:43<00:44,  1.79it/s, loss=20.9, v_num=0]#015Epoch 0:  50%|█████     | 78/156 [00:43<00:43,  1.80it/s, loss=20.9, v_num=0]#015Epoch 0:  50%|█████     | 78/156 [00:43<00:43,  1.80it/s, loss=20.9, v_num=0]\u001b[0m\n",
      "\u001b[34m#015Validating: 0it [00:00, ?it/s]#033[A\u001b[0m\n",
      "\u001b[34m#015Validating:   0%|          | 0/78 [00:00<?, ?it/s]#033[A\u001b[0m\n",
      "\u001b[34m#015Validating:   1%|▏         | 1/78 [00:02<02:42,  2.11s/it]#033[A#015Epoch 0:  51%|█████▏    | 80/156 [00:45<00:43,  1.76it/s, loss=20.9, v_num=0]\u001b[0m\n",
      "\u001b[34m#015Validating:   3%|▎         | 2/78 [00:02<01:15,  1.01it/s]#033[A\u001b[0m\n",
      "\u001b[34m#015Validating:   4%|▍         | 3/78 [00:02<00:46,  1.60it/s]#033[A#015Epoch 0:  53%|█████▎    | 82/156 [00:45<00:41,  1.79it/s, loss=20.9, v_num=0]\u001b[0m\n",
      "\u001b[34m#015Validating:   5%|▌         | 4/78 [00:03<01:04,  1.16it/s]#033[A\u001b[0m\n",
      "\u001b[34m#015Validating:   6%|▋         | 5/78 [00:03<00:45,  1.60it/s]#033[A#015Epoch 0:  54%|█████▍    | 84/156 [00:47<00:40,  1.78it/s, loss=20.9, v_num=0]\u001b[0m\n",
      "\u001b[34m#015Validating:   8%|▊         | 6/78 [00:04<00:34,  2.07it/s]#033[A\u001b[0m\n",
      "\u001b[34m#015Validating:   9%|▉         | 7/78 [00:04<00:29,  2.45it/s]#033[A#015Epoch 0:  55%|█████▌    | 86/156 [00:47<00:38,  1.80it/s, loss=20.9, v_num=0]\u001b[0m\n",
      "\u001b[34m#015Validating:  10%|█         | 8/78 [00:05<00:37,  1.86it/s]#033[A\u001b[0m\n",
      "\u001b[34m#015Validating:  12%|█▏        | 9/78 [00:05<00:29,  2.33it/s]#033[A#015Epoch 0:  56%|█████▋    | 88/156 [00:48<00:37,  1.80it/s, loss=20.9, v_num=0]\u001b[0m\n",
      "\u001b[34m#015Validating:  13%|█▎        | 10/78 [00:05<00:24,  2.78it/s]#033[A\u001b[0m\n",
      "\u001b[34m#015Validating:  14%|█▍        | 11/78 [00:05<00:20,  3.22it/s]#033[A#015Epoch 0:  58%|█████▊    | 90/156 [00:49<00:36,  1.83it/s, loss=20.9, v_num=0]\u001b[0m\n",
      "\u001b[34m#015Validating:  15%|█▌        | 12/78 [00:07<00:43,  1.53it/s]#033[A\u001b[0m\n",
      "\u001b[34m#015Validating:  17%|█▋        | 13/78 [00:07<00:35,  1.81it/s]#033[A#015Epoch 0:  59%|█████▉    | 92/156 [00:50<00:35,  1.81it/s, loss=20.9, v_num=0]\u001b[0m\n",
      "\u001b[34m#015Validating:  18%|█▊        | 14/78 [00:07<00:29,  2.20it/s]#033[A\u001b[0m\n",
      "\u001b[34m#015Validating:  19%|█▉        | 15/78 [00:08<00:24,  2.55it/s]#033[A#015Epoch 0:  60%|██████    | 94/156 [00:51<00:33,  1.83it/s, loss=20.9, v_num=0]\u001b[0m\n",
      "\u001b[34m#015Validating:  21%|██        | 16/78 [00:08<00:22,  2.71it/s]#033[A\u001b[0m\n",
      "\u001b[34m#015Validating:  22%|██▏       | 17/78 [00:08<00:25,  2.38it/s]#033[A#015Epoch 0:  62%|██████▏   | 96/156 [00:52<00:32,  1.84it/s, loss=20.9, v_num=0]\u001b[0m\n",
      "\u001b[34m#015Validating:  23%|██▎       | 18/78 [00:09<00:21,  2.83it/s]#033[A\u001b[0m\n",
      "\u001b[34m#015Validating:  24%|██▍       | 19/78 [00:09<00:17,  3.29it/s]#033[A#015Epoch 0:  63%|██████▎   | 98/156 [00:52<00:31,  1.86it/s, loss=20.9, v_num=0]\u001b[0m\n",
      "\u001b[34m#015Validating:  26%|██▌       | 20/78 [00:09<00:20,  2.89it/s]#033[A\u001b[0m\n",
      "\u001b[34m#015Validating:  27%|██▋       | 21/78 [00:10<00:25,  2.28it/s]#033[A#015Epoch 0:  64%|██████▍   | 100/156 [00:53<00:30,  1.86it/s, loss=20.9, v_num=0]\u001b[0m\n",
      "\u001b[34m#015Validating:  28%|██▊       | 22/78 [00:10<00:20,  2.74it/s]#033[A\u001b[0m\n",
      "\u001b[34m#015Validating:  29%|██▉       | 23/78 [00:10<00:17,  3.22it/s]#033[A#015Epoch 0:  65%|██████▌   | 102/156 [00:54<00:28,  1.88it/s, loss=20.9, v_num=0]\u001b[0m\n",
      "\u001b[34m#015Validating:  31%|███       | 24/78 [00:11<00:19,  2.75it/s]#033[A\u001b[0m\n",
      "\u001b[34m#015Validating:  32%|███▏      | 25/78 [00:11<00:20,  2.61it/s]#033[A#015Epoch 0:  67%|██████▋   | 104/156 [00:55<00:27,  1.89it/s, loss=20.9, v_num=0]\u001b[0m\n",
      "\u001b[34m#015Validating:  33%|███▎      | 26/78 [00:12<00:19,  2.67it/s]#033[A\u001b[0m\n",
      "\u001b[34m#015Validating:  35%|███▍      | 27/78 [00:12<00:16,  3.15it/s]#033[A#015Epoch 0:  68%|██████▊   | 106/156 [00:55<00:26,  1.91it/s, loss=20.9, v_num=0]\u001b[0m\n",
      "\u001b[34m#015Validating:  36%|███▌      | 28/78 [00:12<00:16,  2.96it/s]#033[A\u001b[0m\n",
      "\u001b[34m#015Validating:  37%|███▋      | 29/78 [00:13<00:22,  2.17it/s]#033[A#015Epoch 0:  69%|██████▉   | 108/156 [00:56<00:25,  1.90it/s, loss=20.9, v_num=0]\u001b[0m\n",
      "\u001b[34m#015Validating:  38%|███▊      | 30/78 [00:13<00:18,  2.60it/s]#033[A\u001b[0m\n",
      "\u001b[34m#015Validating:  40%|███▉      | 31/78 [00:13<00:15,  2.96it/s]#033[A#015Epoch 0:  71%|███████   | 110/156 [00:57<00:23,  1.92it/s, loss=20.9, v_num=0]\u001b[0m\n",
      "\u001b[34m#015Validating:  41%|████      | 32/78 [00:14<00:15,  2.99it/s]#033[A\u001b[0m\n",
      "\u001b[34m#015Validating:  42%|████▏     | 33/78 [00:14<00:19,  2.30it/s]#033[A#015Epoch 0:  72%|███████▏  | 112/156 [00:58<00:22,  1.93it/s, loss=20.9, v_num=0]\u001b[0m\n",
      "\u001b[34m#015Validating:  44%|████▎     | 34/78 [00:14<00:15,  2.75it/s]#033[A\u001b[0m\n",
      "\u001b[34m#015Validating:  45%|████▍     | 35/78 [00:15<00:14,  2.93it/s]#033[A#015Epoch 0:  73%|███████▎  | 114/156 [00:58<00:21,  1.94it/s, loss=20.9, v_num=0]\u001b[0m\n",
      "\u001b[34m#015Validating:  46%|████▌     | 36/78 [00:15<00:16,  2.53it/s]#033[A\u001b[0m\n",
      "\u001b[34m#015Validating:  47%|████▋     | 37/78 [00:16<00:15,  2.60it/s]#033[A#015Epoch 0:  74%|███████▍  | 116/156 [00:59<00:20,  1.95it/s, loss=20.9, v_num=0]\u001b[0m\n",
      "\u001b[34m#015Validating:  49%|████▊     | 38/78 [00:16<00:14,  2.83it/s]#033[A\u001b[0m\n",
      "\u001b[34m#015Validating:  50%|█████     | 39/78 [00:16<00:12,  3.10it/s]#033[A#015Epoch 0:  76%|███████▌  | 118/156 [01:00<00:19,  1.96it/s, loss=20.9, v_num=0]\u001b[0m\n",
      "\u001b[34m#015Validating:  51%|█████▏    | 40/78 [00:17<00:14,  2.63it/s]#033[A\u001b[0m\n",
      "\u001b[34m#015Validating:  53%|█████▎    | 41/78 [00:17<00:14,  2.62it/s]#033[A#015Epoch 0:  77%|███████▋  | 120/156 [01:00<00:18,  1.97it/s, loss=20.9, v_num=0]\u001b[0m\n",
      "\u001b[34m#015Validating:  54%|█████▍    | 42/78 [00:17<00:13,  2.65it/s]#033[A\u001b[0m\n",
      "\u001b[34m#015Validating:  55%|█████▌    | 43/78 [00:18<00:11,  3.11it/s]#033[A#015Epoch 0:  78%|███████▊  | 122/156 [01:01<00:17,  1.98it/s, loss=20.9, v_num=0]\u001b[0m\n",
      "\u001b[34m#015Validating:  56%|█████▋    | 44/78 [00:18<00:12,  2.72it/s]#033[A\u001b[0m\n",
      "\u001b[34m#015Validating:  58%|█████▊    | 45/78 [00:18<00:11,  2.97it/s]#033[A#015Epoch 0:  79%|███████▉  | 124/156 [01:02<00:16,  1.99it/s, loss=20.9, v_num=0]\u001b[0m\n",
      "\u001b[34m#015Validating:  59%|█████▉    | 46/78 [00:19<00:11,  2.83it/s]#033[A\u001b[0m\n",
      "\u001b[34m#015Validating:  60%|██████    | 47/78 [00:19<00:09,  3.17it/s]#033[A#015Epoch 0:  81%|████████  | 126/156 [01:02<00:14,  2.00it/s, loss=20.9, v_num=0]\u001b[0m\n",
      "\u001b[34m#015Validating:  62%|██████▏   | 48/78 [00:19<00:11,  2.71it/s]#033[A\u001b[0m\n",
      "\u001b[34m#015Validating:  63%|██████▎   | 49/78 [00:20<00:09,  3.15it/s]#033[A#015Epoch 0:  82%|████████▏ | 128/156 [01:03<00:13,  2.01it/s, loss=20.9, v_num=0]\u001b[0m\n",
      "\u001b[34m#015Validating:  64%|██████▍   | 50/78 [00:20<00:11,  2.42it/s]#033[A\u001b[0m\n",
      "\u001b[34m#015Validating:  65%|██████▌   | 51/78 [00:21<00:10,  2.59it/s]#033[A#015Epoch 0:  83%|████████▎ | 130/156 [01:04<00:12,  2.01it/s, loss=20.9, v_num=0]\u001b[0m\n",
      "\u001b[34m#015Validating:  67%|██████▋   | 52/78 [00:21<00:09,  2.66it/s]#033[A\u001b[0m\n",
      "\u001b[34m#015Validating:  68%|██████▊   | 53/78 [00:21<00:08,  3.05it/s]#033[A#015Epoch 0:  85%|████████▍ | 132/156 [01:05<00:11,  2.03it/s, loss=20.9, v_num=0]\u001b[0m\n",
      "\u001b[34m#015Validating:  69%|██████▉   | 54/78 [00:22<00:09,  2.46it/s]#033[A\u001b[0m\n",
      "\u001b[34m#015Validating:  71%|███████   | 55/78 [00:22<00:08,  2.70it/s]#033[A#015Epoch 0:  86%|████████▌ | 134/156 [01:05<00:10,  2.03it/s, loss=20.9, v_num=0]\u001b[0m\n",
      "\u001b[34m#015Validating:  72%|███████▏  | 56/78 [00:22<00:07,  3.01it/s]#033[A\u001b[0m\n",
      "\u001b[34m#015Validating:  73%|███████▎  | 57/78 [00:23<00:06,  3.40it/s]#033[A#015Epoch 0:  87%|████████▋ | 136/156 [01:06<00:09,  2.05it/s, loss=20.9, v_num=0]\u001b[0m\n",
      "\u001b[34m#015Validating:  74%|███████▍  | 58/78 [00:23<00:07,  2.66it/s]#033[A\u001b[0m\n",
      "\u001b[34m#015Validating:  76%|███████▌  | 59/78 [00:23<00:06,  2.85it/s]#033[A#015Epoch 0:  88%|████████▊ | 138/156 [01:07<00:08,  2.05it/s, loss=20.9, v_num=0]\u001b[0m\n",
      "\u001b[34m#015Validating:  77%|███████▋  | 60/78 [00:24<00:06,  2.96it/s]#033[A\u001b[0m\n",
      "\u001b[34m#015Validating:  78%|███████▊  | 61/78 [00:24<00:04,  3.41it/s]#033[A#015Epoch 0:  90%|████████▉ | 140/156 [01:07<00:07,  2.07it/s, loss=20.9, v_num=0]\u001b[0m\n",
      "\u001b[34m#015Validating:  79%|███████▉  | 62/78 [00:25<00:07,  2.12it/s]#033[A\u001b[0m\n",
      "\u001b[34m#015Validating:  81%|████████  | 63/78 [00:25<00:06,  2.30it/s]#033[A#015Epoch 0:  91%|█████████ | 142/156 [01:09<00:06,  2.06it/s, loss=20.9, v_num=0]\u001b[0m\n",
      "\u001b[34m#015Validating:  82%|████████▏ | 64/78 [00:25<00:05,  2.66it/s]#033[A\u001b[0m\n",
      "\u001b[34m#015Validating:  83%|████████▎ | 65/78 [00:26<00:04,  2.66it/s]#033[A#015Epoch 0:  92%|█████████▏| 144/156 [01:09<00:05,  2.07it/s, loss=20.9, v_num=0]\u001b[0m\n",
      "\u001b[34m#015Validating:  85%|████████▍ | 66/78 [00:26<00:04,  2.92it/s]#033[A\u001b[0m\n",
      "\u001b[34m#015Validating:  86%|████████▌ | 67/78 [00:26<00:03,  2.80it/s]#033[A#015Epoch 0:  94%|█████████▎| 146/156 [01:10<00:04,  2.08it/s, loss=20.9, v_num=0]\u001b[0m\n",
      "\u001b[34m#015Validating:  87%|████████▋ | 68/78 [00:27<00:03,  3.19it/s]#033[A\u001b[0m\n",
      "\u001b[34m#015Validating:  88%|████████▊ | 69/78 [00:27<00:02,  3.63it/s]#033[A#015Epoch 0:  95%|█████████▍| 148/156 [01:10<00:03,  2.09it/s, loss=20.9, v_num=0]\u001b[0m\n",
      "\u001b[34m#015Validating:  90%|████████▉ | 70/78 [00:27<00:02,  2.69it/s]#033[A\u001b[0m\n",
      "\u001b[34m#015Validating:  91%|█████████ | 71/78 [00:28<00:02,  2.86it/s]#033[A#015Epoch 0:  96%|█████████▌| 150/156 [01:11<00:02,  2.10it/s, loss=20.9, v_num=0]\u001b[0m\n",
      "\u001b[34m#015Validating:  92%|█████████▏| 72/78 [00:28<00:01,  3.30it/s]#033[A\u001b[0m\n",
      "\u001b[34m#015Validating:  94%|█████████▎| 73/78 [00:28<00:01,  3.62it/s]#033[A#015Epoch 0:  97%|█████████▋| 152/156 [01:11<00:01,  2.11it/s, loss=20.9, v_num=0]\u001b[0m\n",
      "\u001b[34m#015Validating:  95%|█████████▍| 74/78 [00:29<00:01,  2.67it/s]#033[A\u001b[0m\n",
      "\u001b[34m#015Validating:  96%|█████████▌| 75/78 [00:29<00:00,  3.19it/s]#033[A#015Epoch 0:  99%|█████████▊| 154/156 [01:12<00:00,  2.12it/s, loss=20.9, v_num=0]\u001b[0m\n",
      "\u001b[34m#015Validating:  97%|█████████▋| 76/78 [00:29<00:00,  3.70it/s]#033[A\u001b[0m\n",
      "\u001b[34m#015Validating:  99%|█████████▊| 77/78 [00:29<00:00,  4.14it/s]#033[A#015Epoch 0: 100%|██████████| 156/156 [01:13<00:00,  2.13it/s, loss=20.9, v_num=0]\u001b[0m\n",
      "\u001b[34m#015Validating: 100%|██████████| 78/78 [00:29<00:00,  4.43it/s]#033[A#015Epoch 0: 100%|██████████| 156/156 [01:13<00:00,  2.13it/s, loss=20.9, v_num=0]\u001b[0m\n"
     ]
    }
   ],
   "source": [
    "from sagemaker.inputs import FileSystemInput\n",
    "file_system_input = FileSystemInput(file_system_id='fs-025101e72b1cc96ca',\n",
    "                                    file_system_type='EFS',\n",
    "                                    directory_path='/',\n",
    "                                    file_system_access_mode='ro')\n",
    "data_channels = {\"training\": file_system_input}\n",
    "# loc = \"s3://smokeynet-reduced-dataset\"\n",
    "# channels = {\"training\": loc}\n",
    "# estimator.fit(inputs=channels)\n",
    "estimator.fit(inputs=data_channels)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "b339f68d",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "c451999b",
   "metadata": {},
   "outputs": [
    {
     "ename": "NameError",
     "evalue": "name 'pt_mnist_model_data' is not defined",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mNameError\u001b[0m                                 Traceback (most recent call last)",
      "\u001b[0;32m/tmp/ipykernel_12443/2980632821.py\u001b[0m in \u001b[0;36m<cell line: 2>\u001b[0;34m()\u001b[0m\n\u001b[1;32m      1\u001b[0m \u001b[0;31m###### pt_mnist_model_data = est.model_data\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 2\u001b[0;31m \u001b[0mprint\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m\"Model artifact saved at:\\n\"\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mpt_mnist_model_data\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m",
      "\u001b[0;31mNameError\u001b[0m: name 'pt_mnist_model_data' is not defined"
     ]
    }
   ],
   "source": [
    "###### pt_mnist_model_data = est.model_data\n",
    "print(\"Model artifact saved at:\\n\", pt_mnist_model_data)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "be3db68c",
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "os.system('ls -l /home/ec2-user')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "f9192655",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "conda_pytorch_p38",
   "language": "python",
   "name": "conda_pytorch_p38"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.12"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}