002_Train_Model_101.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/javascript": [
       "\n",
       "            setTimeout(function() {\n",
       "                var nbb_cell_id = 1;\n",
       "                var nbb_unformatted_code = \"%load_ext nb_black\";\n",
       "                var nbb_formatted_code = \"%load_ext nb_black\";\n",
       "                var nbb_cells = Jupyter.notebook.get_cells();\n",
       "                for (var i = 0; i < nbb_cells.length; ++i) {\n",
       "                    if (nbb_cells[i].input_prompt_number == nbb_cell_id) {\n",
       "                        if (nbb_cells[i].get_text() == nbb_unformatted_code) {\n",
       "                             nbb_cells[i].set_text(nbb_formatted_code);\n",
       "                        }\n",
       "                        break;\n",
       "                    }\n",
       "                }\n",
       "            }, 500);\n",
       "            "
      ],
      "text/plain": [
       "<IPython.core.display.Javascript object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "%load_ext nb_black"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/javascript": [
       "\n",
       "            setTimeout(function() {\n",
       "                var nbb_cell_id = 2;\n",
       "                var nbb_unformatted_code = \"import os\\nimport logging\\nfrom datetime import datetime\";\n",
       "                var nbb_formatted_code = \"import os\\nimport logging\\nfrom datetime import datetime\";\n",
       "                var nbb_cells = Jupyter.notebook.get_cells();\n",
       "                for (var i = 0; i < nbb_cells.length; ++i) {\n",
       "                    if (nbb_cells[i].input_prompt_number == nbb_cell_id) {\n",
       "                        if (nbb_cells[i].get_text() == nbb_unformatted_code) {\n",
       "                             nbb_cells[i].set_text(nbb_formatted_code);\n",
       "                        }\n",
       "                        break;\n",
       "                    }\n",
       "                }\n",
       "            }, 500);\n",
       "            "
      ],
      "text/plain": [
       "<IPython.core.display.Javascript object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "import os\n",
    "import logging\n",
    "from datetime import datetime"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/javascript": [
       "\n",
       "            setTimeout(function() {\n",
       "                var nbb_cell_id = 3;\n",
       "                var nbb_unformatted_code = \"import boto3\\nimport sagemaker\\nfrom sagemaker.session import TrainingInput\\nfrom sagemaker import image_uris\\nfrom sagemaker import hyperparameters\";\n",
       "                var nbb_formatted_code = \"import boto3\\nimport sagemaker\\nfrom sagemaker.session import TrainingInput\\nfrom sagemaker import image_uris\\nfrom sagemaker import hyperparameters\";\n",
       "                var nbb_cells = Jupyter.notebook.get_cells();\n",
       "                for (var i = 0; i < nbb_cells.length; ++i) {\n",
       "                    if (nbb_cells[i].input_prompt_number == nbb_cell_id) {\n",
       "                        if (nbb_cells[i].get_text() == nbb_unformatted_code) {\n",
       "                             nbb_cells[i].set_text(nbb_formatted_code);\n",
       "                        }\n",
       "                        break;\n",
       "                    }\n",
       "                }\n",
       "            }, 500);\n",
       "            "
      ],
      "text/plain": [
       "<IPython.core.display.Javascript object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "import boto3\n",
    "import sagemaker\n",
    "from sagemaker.session import TrainingInput\n",
    "from sagemaker import image_uris\n",
    "from sagemaker import hyperparameters"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/javascript": [
       "\n",
       "            setTimeout(function() {\n",
       "                var nbb_cell_id = 4;\n",
       "                var nbb_unformatted_code = \"boto3.set_stream_logger(name=\\\"botocore.credentials\\\", level=logging.WARNING)\";\n",
       "                var nbb_formatted_code = \"boto3.set_stream_logger(name=\\\"botocore.credentials\\\", level=logging.WARNING)\";\n",
       "                var nbb_cells = Jupyter.notebook.get_cells();\n",
       "                for (var i = 0; i < nbb_cells.length; ++i) {\n",
       "                    if (nbb_cells[i].input_prompt_number == nbb_cell_id) {\n",
       "                        if (nbb_cells[i].get_text() == nbb_unformatted_code) {\n",
       "                             nbb_cells[i].set_text(nbb_formatted_code);\n",
       "                        }\n",
       "                        break;\n",
       "                    }\n",
       "                }\n",
       "            }, 500);\n",
       "            "
      ],
      "text/plain": [
       "<IPython.core.display.Javascript object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "boto3.set_stream_logger(name=\"botocore.credentials\", level=logging.WARNING)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "ap-southeast-2\n"
     ]
    },
    {
     "data": {
      "application/javascript": [
       "\n",
       "            setTimeout(function() {\n",
       "                var nbb_cell_id = 5;\n",
       "                var nbb_unformatted_code = \"region = sagemaker.Session().boto_region_name\\nprint(region)\";\n",
       "                var nbb_formatted_code = \"region = sagemaker.Session().boto_region_name\\nprint(region)\";\n",
       "                var nbb_cells = Jupyter.notebook.get_cells();\n",
       "                for (var i = 0; i < nbb_cells.length; ++i) {\n",
       "                    if (nbb_cells[i].input_prompt_number == nbb_cell_id) {\n",
       "                        if (nbb_cells[i].get_text() == nbb_unformatted_code) {\n",
       "                             nbb_cells[i].set_text(nbb_formatted_code);\n",
       "                        }\n",
       "                        break;\n",
       "                    }\n",
       "                }\n",
       "            }, 500);\n",
       "            "
      ],
      "text/plain": [
       "<IPython.core.display.Javascript object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "region = sagemaker.Session().boto_region_name\n",
    "print(region)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/javascript": [
       "\n",
       "            setTimeout(function() {\n",
       "                var nbb_cell_id = 6;\n",
       "                var nbb_unformatted_code = \"# role_arn = sagemaker.get_execution_role()\\nrole_arn = os.getenv(\\\"SGMKR_ROLE_ARN\\\")\";\n",
       "                var nbb_formatted_code = \"# role_arn = sagemaker.get_execution_role()\\nrole_arn = os.getenv(\\\"SGMKR_ROLE_ARN\\\")\";\n",
       "                var nbb_cells = Jupyter.notebook.get_cells();\n",
       "                for (var i = 0; i < nbb_cells.length; ++i) {\n",
       "                    if (nbb_cells[i].input_prompt_number == nbb_cell_id) {\n",
       "                        if (nbb_cells[i].get_text() == nbb_unformatted_code) {\n",
       "                             nbb_cells[i].set_text(nbb_formatted_code);\n",
       "                        }\n",
       "                        break;\n",
       "                    }\n",
       "                }\n",
       "            }, 500);\n",
       "            "
      ],
      "text/plain": [
       "<IPython.core.display.Javascript object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "# role_arn = sagemaker.get_execution_role()\n",
    "role_arn = os.getenv(\"SGMKR_ROLE_ARN\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/javascript": [
       "\n",
       "            setTimeout(function() {\n",
       "                var nbb_cell_id = 7;\n",
       "                var nbb_unformatted_code = \"bucket = \\\"sgmkr-course\\\"\\nprefix = \\\"iris\\\"\";\n",
       "                var nbb_formatted_code = \"bucket = \\\"sgmkr-course\\\"\\nprefix = \\\"iris\\\"\";\n",
       "                var nbb_cells = Jupyter.notebook.get_cells();\n",
       "                for (var i = 0; i < nbb_cells.length; ++i) {\n",
       "                    if (nbb_cells[i].input_prompt_number == nbb_cell_id) {\n",
       "                        if (nbb_cells[i].get_text() == nbb_unformatted_code) {\n",
       "                             nbb_cells[i].set_text(nbb_formatted_code);\n",
       "                        }\n",
       "                        break;\n",
       "                    }\n",
       "                }\n",
       "            }, 500);\n",
       "            "
      ],
      "text/plain": [
       "<IPython.core.display.Javascript object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "bucket = \"sgmkr-course\"\n",
    "prefix = \"iris\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "                           PRE batch_transform/\r\n",
      "                           PRE data/\r\n",
      "                           PRE model/\r\n",
      "2023-02-03 13:15:44          0 \r\n"
     ]
    },
    {
     "data": {
      "application/javascript": [
       "\n",
       "            setTimeout(function() {\n",
       "                var nbb_cell_id = 8;\n",
       "                var nbb_unformatted_code = \"!aws s3 ls {bucket}/{prefix}/\";\n",
       "                var nbb_formatted_code = \"!aws s3 ls {bucket}/{prefix}/\";\n",
       "                var nbb_cells = Jupyter.notebook.get_cells();\n",
       "                for (var i = 0; i < nbb_cells.length; ++i) {\n",
       "                    if (nbb_cells[i].input_prompt_number == nbb_cell_id) {\n",
       "                        if (nbb_cells[i].get_text() == nbb_unformatted_code) {\n",
       "                             nbb_cells[i].set_text(nbb_formatted_code);\n",
       "                        }\n",
       "                        break;\n",
       "                    }\n",
       "                }\n",
       "            }, 500);\n",
       "            "
      ],
      "text/plain": [
       "<IPython.core.display.Javascript object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "!aws s3 ls {bucket}/{prefix}/"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2023-02-03 13:16:32          0 iris/data/\r\n",
      "2023-03-16 18:09:28        900 iris/data/iris_test.csv\r\n",
      "2023-03-16 18:09:27       1800 iris/data/iris_train.csv\r\n"
     ]
    },
    {
     "data": {
      "application/javascript": [
       "\n",
       "            setTimeout(function() {\n",
       "                var nbb_cell_id = 9;\n",
       "                var nbb_unformatted_code = \"!aws s3 ls {bucket}/{prefix}/data/ --recursive\";\n",
       "                var nbb_formatted_code = \"!aws s3 ls {bucket}/{prefix}/data/ --recursive\";\n",
       "                var nbb_cells = Jupyter.notebook.get_cells();\n",
       "                for (var i = 0; i < nbb_cells.length; ++i) {\n",
       "                    if (nbb_cells[i].input_prompt_number == nbb_cell_id) {\n",
       "                        if (nbb_cells[i].get_text() == nbb_unformatted_code) {\n",
       "                             nbb_cells[i].set_text(nbb_formatted_code);\n",
       "                        }\n",
       "                        break;\n",
       "                    }\n",
       "                }\n",
       "            }, 500);\n",
       "            "
      ],
      "text/plain": [
       "<IPython.core.display.Javascript object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "!aws s3 ls {bucket}/{prefix}/data/ --recursive"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train file uri: s3://sgmkr-course/iris/data/iris_train.csv\n",
      "valid file uri: s3://sgmkr-course/iris/data/iris_test.csv\n"
     ]
    },
    {
     "data": {
      "application/javascript": [
       "\n",
       "            setTimeout(function() {\n",
       "                var nbb_cell_id = 10;\n",
       "                var nbb_unformatted_code = \"train_file = \\\"data/iris_train.csv\\\"\\nvalid_file = \\\"data/iris_test.csv\\\"\\n\\ntrain_file_uri = \\\"s3://{}/{}/{}\\\".format(bucket, prefix, train_file)\\nvalid_file_uri = \\\"s3://{}/{}/{}\\\".format(bucket, prefix, valid_file)\\nprint(\\\"train file uri:\\\", train_file_uri)\\nprint(\\\"valid file uri:\\\", valid_file_uri)\";\n",
       "                var nbb_formatted_code = \"train_file = \\\"data/iris_train.csv\\\"\\nvalid_file = \\\"data/iris_test.csv\\\"\\n\\ntrain_file_uri = \\\"s3://{}/{}/{}\\\".format(bucket, prefix, train_file)\\nvalid_file_uri = \\\"s3://{}/{}/{}\\\".format(bucket, prefix, valid_file)\\nprint(\\\"train file uri:\\\", train_file_uri)\\nprint(\\\"valid file uri:\\\", valid_file_uri)\";\n",
       "                var nbb_cells = Jupyter.notebook.get_cells();\n",
       "                for (var i = 0; i < nbb_cells.length; ++i) {\n",
       "                    if (nbb_cells[i].input_prompt_number == nbb_cell_id) {\n",
       "                        if (nbb_cells[i].get_text() == nbb_unformatted_code) {\n",
       "                             nbb_cells[i].set_text(nbb_formatted_code);\n",
       "                        }\n",
       "                        break;\n",
       "                    }\n",
       "                }\n",
       "            }, 500);\n",
       "            "
      ],
      "text/plain": [
       "<IPython.core.display.Javascript object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "train_file = \"data/iris_train.csv\"\n",
    "valid_file = \"data/iris_test.csv\"\n",
    "\n",
    "train_file_uri = \"s3://{}/{}/{}\".format(bucket, prefix, train_file)\n",
    "valid_file_uri = \"s3://{}/{}/{}\".format(bucket, prefix, valid_file)\n",
    "print(\"train file uri:\", train_file_uri)\n",
    "print(\"valid file uri:\", valid_file_uri)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<sagemaker.inputs.TrainingInput object at 0x7fedc9cb8130>\n"
     ]
    },
    {
     "data": {
      "application/javascript": [
       "\n",
       "            setTimeout(function() {\n",
       "                var nbb_cell_id = 11;\n",
       "                var nbb_unformatted_code = \"train_ip = TrainingInput(train_file_uri, content_type=\\\"csv\\\")\\nprint(train_ip)\";\n",
       "                var nbb_formatted_code = \"train_ip = TrainingInput(train_file_uri, content_type=\\\"csv\\\")\\nprint(train_ip)\";\n",
       "                var nbb_cells = Jupyter.notebook.get_cells();\n",
       "                for (var i = 0; i < nbb_cells.length; ++i) {\n",
       "                    if (nbb_cells[i].input_prompt_number == nbb_cell_id) {\n",
       "                        if (nbb_cells[i].get_text() == nbb_unformatted_code) {\n",
       "                             nbb_cells[i].set_text(nbb_formatted_code);\n",
       "                        }\n",
       "                        break;\n",
       "                    }\n",
       "                }\n",
       "            }, 500);\n",
       "            "
      ],
      "text/plain": [
       "<IPython.core.display.Javascript object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "train_ip = TrainingInput(train_file_uri, content_type=\"csv\")\n",
    "print(train_ip)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<sagemaker.inputs.TrainingInput object at 0x7fedbc881df0>\n"
     ]
    },
    {
     "data": {
      "application/javascript": [
       "\n",
       "            setTimeout(function() {\n",
       "                var nbb_cell_id = 12;\n",
       "                var nbb_unformatted_code = \"valid_ip = TrainingInput(valid_file_uri, content_type=\\\"csv\\\")\\nprint(valid_ip)\";\n",
       "                var nbb_formatted_code = \"valid_ip = TrainingInput(valid_file_uri, content_type=\\\"csv\\\")\\nprint(valid_ip)\";\n",
       "                var nbb_cells = Jupyter.notebook.get_cells();\n",
       "                for (var i = 0; i < nbb_cells.length; ++i) {\n",
       "                    if (nbb_cells[i].input_prompt_number == nbb_cell_id) {\n",
       "                        if (nbb_cells[i].get_text() == nbb_unformatted_code) {\n",
       "                             nbb_cells[i].set_text(nbb_formatted_code);\n",
       "                        }\n",
       "                        break;\n",
       "                    }\n",
       "                }\n",
       "            }, 500);\n",
       "            "
      ],
      "text/plain": [
       "<IPython.core.display.Javascript object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "valid_ip = TrainingInput(valid_file_uri, content_type=\"csv\")\n",
    "print(valid_ip)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "s3://sgmkr-course/iris/model\n"
     ]
    },
    {
     "data": {
      "application/javascript": [
       "\n",
       "            setTimeout(function() {\n",
       "                var nbb_cell_id = 13;\n",
       "                var nbb_unformatted_code = \"model_op = \\\"s3://{}/{}/{}\\\".format(bucket, prefix, \\\"model\\\")\\nprint(model_op)\";\n",
       "                var nbb_formatted_code = \"model_op = \\\"s3://{}/{}/{}\\\".format(bucket, prefix, \\\"model\\\")\\nprint(model_op)\";\n",
       "                var nbb_cells = Jupyter.notebook.get_cells();\n",
       "                for (var i = 0; i < nbb_cells.length; ++i) {\n",
       "                    if (nbb_cells[i].input_prompt_number == nbb_cell_id) {\n",
       "                        if (nbb_cells[i].get_text() == nbb_unformatted_code) {\n",
       "                             nbb_cells[i].set_text(nbb_formatted_code);\n",
       "                        }\n",
       "                        break;\n",
       "                    }\n",
       "                }\n",
       "            }, 500);\n",
       "            "
      ],
      "text/plain": [
       "<IPython.core.display.Javascript object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "model_op = \"s3://{}/{}/{}\".format(bucket, prefix, \"model\")\n",
    "print(model_op)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Ignoring unnecessary instance type: None.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "544295431143.dkr.ecr.ap-southeast-2.amazonaws.com/xgboost:latest\n"
     ]
    },
    {
     "data": {
      "application/javascript": [
       "\n",
       "            setTimeout(function() {\n",
       "                var nbb_cell_id = 14;\n",
       "                var nbb_unformatted_code = \"model_img = sagemaker.image_uris.retrieve(\\\"xgboost\\\", region, \\\"latest\\\")\\nprint(model_img)\";\n",
       "                var nbb_formatted_code = \"model_img = sagemaker.image_uris.retrieve(\\\"xgboost\\\", region, \\\"latest\\\")\\nprint(model_img)\";\n",
       "                var nbb_cells = Jupyter.notebook.get_cells();\n",
       "                for (var i = 0; i < nbb_cells.length; ++i) {\n",
       "                    if (nbb_cells[i].input_prompt_number == nbb_cell_id) {\n",
       "                        if (nbb_cells[i].get_text() == nbb_unformatted_code) {\n",
       "                             nbb_cells[i].set_text(nbb_formatted_code);\n",
       "                        }\n",
       "                        break;\n",
       "                    }\n",
       "                }\n",
       "            }, 500);\n",
       "            "
      ],
      "text/plain": [
       "<IPython.core.display.Javascript object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "model_img = sagemaker.image_uris.retrieve(\"xgboost\", region, \"latest\")\n",
    "print(model_img)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/javascript": [
       "\n",
       "            setTimeout(function() {\n",
       "                var nbb_cell_id = 15;\n",
       "                var nbb_unformatted_code = \"base_job_name = 'iris-xgboost'\";\n",
       "                var nbb_formatted_code = \"base_job_name = \\\"iris-xgboost\\\"\";\n",
       "                var nbb_cells = Jupyter.notebook.get_cells();\n",
       "                for (var i = 0; i < nbb_cells.length; ++i) {\n",
       "                    if (nbb_cells[i].input_prompt_number == nbb_cell_id) {\n",
       "                        if (nbb_cells[i].get_text() == nbb_unformatted_code) {\n",
       "                             nbb_cells[i].set_text(nbb_formatted_code);\n",
       "                        }\n",
       "                        break;\n",
       "                    }\n",
       "                }\n",
       "            }, 500);\n",
       "            "
      ],
      "text/plain": [
       "<IPython.core.display.Javascript object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "base_job_name = \"iris-xgboost-\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/javascript": [
       "\n",
       "            setTimeout(function() {\n",
       "                var nbb_cell_id = 16;\n",
       "                var nbb_unformatted_code = \"xgb_model=sagemaker.estimator.Estimator(\\n    image_uri=model_img,\\n    role=role_arn,\\n    base_job_name=base_job_name,\\n    instance_count=1,\\n    instance_type='ml.m4.xlarge',\\n    output_path=model_op,\\n    sagemaker_session=sagemaker.Session(),\\n    volume_size=5,\\n)\";\n",
       "                var nbb_formatted_code = \"xgb_model = sagemaker.estimator.Estimator(\\n    image_uri=model_img,\\n    role=role_arn,\\n    base_job_name=base_job_name,\\n    instance_count=1,\\n    instance_type=\\\"ml.m4.xlarge\\\",\\n    output_path=model_op,\\n    sagemaker_session=sagemaker.Session(),\\n    volume_size=5,\\n)\";\n",
       "                var nbb_cells = Jupyter.notebook.get_cells();\n",
       "                for (var i = 0; i < nbb_cells.length; ++i) {\n",
       "                    if (nbb_cells[i].input_prompt_number == nbb_cell_id) {\n",
       "                        if (nbb_cells[i].get_text() == nbb_unformatted_code) {\n",
       "                             nbb_cells[i].set_text(nbb_formatted_code);\n",
       "                        }\n",
       "                        break;\n",
       "                    }\n",
       "                }\n",
       "            }, 500);\n",
       "            "
      ],
      "text/plain": [
       "<IPython.core.display.Javascript object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "xgb_model = sagemaker.estimator.Estimator(\n",
    "    image_uri=model_img,\n",
    "    role=role_arn,\n",
    "    base_job_name=base_job_name,\n",
    "    instance_count=1,\n",
    "    instance_type=\"ml.m4.xlarge\",\n",
    "    output_path=model_op,\n",
    "    sagemaker_session=sagemaker.Session(),\n",
    "    volume_size=5,\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/javascript": [
       "\n",
       "            setTimeout(function() {\n",
       "                var nbb_cell_id = 17;\n",
       "                var nbb_unformatted_code = \"xgb_model.set_hyperparameters(\\n    num_class = 3,\\n    max_depth = 5,\\n    num_round = 10,\\n    objective = \\\"multi:softmax\\\",\\n)\";\n",
       "                var nbb_formatted_code = \"xgb_model.set_hyperparameters(\\n    num_class=3, max_depth=5, num_round=10, objective=\\\"multi:softmax\\\",\\n)\";\n",
       "                var nbb_cells = Jupyter.notebook.get_cells();\n",
       "                for (var i = 0; i < nbb_cells.length; ++i) {\n",
       "                    if (nbb_cells[i].input_prompt_number == nbb_cell_id) {\n",
       "                        if (nbb_cells[i].get_text() == nbb_unformatted_code) {\n",
       "                             nbb_cells[i].set_text(nbb_formatted_code);\n",
       "                        }\n",
       "                        break;\n",
       "                    }\n",
       "                }\n",
       "            }, 500);\n",
       "            "
      ],
      "text/plain": [
       "<IPython.core.display.Javascript object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "xgb_model.set_hyperparameters(\n",
    "    num_class=3, max_depth=5, num_round=10, objective=\"multi:softmax\",\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "iris-xgboost2023-03-22-18-09-35\n"
     ]
    },
    {
     "data": {
      "application/javascript": [
       "\n",
       "            setTimeout(function() {\n",
       "                var nbb_cell_id = 18;\n",
       "                var nbb_unformatted_code = \"job_name = base_job_name + datetime.today().strftime('%Y-%m-%d-%H-%M-%S')\\nprint(job_name)\";\n",
       "                var nbb_formatted_code = \"job_name = base_job_name + datetime.today().strftime(\\\"%Y-%m-%d-%H-%M-%S\\\")\\nprint(job_name)\";\n",
       "                var nbb_cells = Jupyter.notebook.get_cells();\n",
       "                for (var i = 0; i < nbb_cells.length; ++i) {\n",
       "                    if (nbb_cells[i].input_prompt_number == nbb_cell_id) {\n",
       "                        if (nbb_cells[i].get_text() == nbb_unformatted_code) {\n",
       "                             nbb_cells[i].set_text(nbb_formatted_code);\n",
       "                        }\n",
       "                        break;\n",
       "                    }\n",
       "                }\n",
       "            }, 500);\n",
       "            "
      ],
      "text/plain": [
       "<IPython.core.display.Javascript object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "job_name = base_job_name + datetime.today().strftime(\"%Y-%m-%d-%H-%M-%S\")\n",
    "print(job_name)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Creating training-job with name: iris-xgboost2023-03-22-18-09-35\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2023-03-22 07:09:37 Starting - Starting the training job...\n",
      "2023-03-22 07:10:02 Starting - Preparing the instances for training......\n",
      "2023-03-22 07:11:02 Downloading - Downloading input data...\n",
      "2023-03-22 07:11:32 Training - Downloading the training image...\n",
      "2023-03-22 07:12:02 Training - Training image download completed. Training in progress..\u001b[34mArguments: train\u001b[0m\n",
      "\u001b[34m[2023-03-22:07:12:17:INFO] Running standalone xgboost training.\u001b[0m\n",
      "\u001b[34m[2023-03-22:07:12:17:INFO] File size need to be processed in the node: 0.0mb. Available memory size in the node: 8604.3mb\u001b[0m\n",
      "\u001b[34m[2023-03-22:07:12:17:INFO] Determined delimiter of CSV input is ','\u001b[0m\n",
      "\u001b[34m[07:12:17] S3DistributionType set as FullyReplicated\u001b[0m\n",
      "\u001b[34m[07:12:17] 100x4 matrix with 400 entries loaded from /opt/ml/input/data/train?format=csv&label_column=0&delimiter=,\u001b[0m\n",
      "\u001b[34m[2023-03-22:07:12:17:INFO] Determined delimiter of CSV input is ','\u001b[0m\n",
      "\u001b[34m[07:12:17] S3DistributionType set as FullyReplicated\u001b[0m\n",
      "\u001b[34m[07:12:17] 50x4 matrix with 200 entries loaded from /opt/ml/input/data/validation?format=csv&label_column=0&delimiter=,\u001b[0m\n",
      "\u001b[34m[07:12:17] src/tree/updater_prune.cc:74: tree pruning end, 1 roots, 2 extra nodes, 0 pruned nodes, max_depth=1\u001b[0m\n",
      "\u001b[34m[07:12:17] src/tree/updater_prune.cc:74: tree pruning end, 1 roots, 8 extra nodes, 0 pruned nodes, max_depth=3\u001b[0m\n",
      "\u001b[34m[07:12:17] src/tree/updater_prune.cc:74: tree pruning end, 1 roots, 4 extra nodes, 0 pruned nodes, max_depth=2\u001b[0m\n",
      "\u001b[34m[0]#011train-merror:0.03#011validation-merror:0.06\u001b[0m\n",
      "\u001b[34m[07:12:17] src/tree/updater_prune.cc:74: tree pruning end, 1 roots, 2 extra nodes, 0 pruned nodes, max_depth=1\u001b[0m\n",
      "\u001b[34m[07:12:17] src/tree/updater_prune.cc:74: tree pruning end, 1 roots, 8 extra nodes, 0 pruned nodes, max_depth=3\u001b[0m\n",
      "\u001b[34m[07:12:17] src/tree/updater_prune.cc:74: tree pruning end, 1 roots, 6 extra nodes, 0 pruned nodes, max_depth=2\u001b[0m\n",
      "\u001b[34m[1]#011train-merror:0.02#011validation-merror:0.02\u001b[0m\n",
      "\u001b[34m[07:12:17] src/tree/updater_prune.cc:74: tree pruning end, 1 roots, 2 extra nodes, 0 pruned nodes, max_depth=1\u001b[0m\n",
      "\u001b[34m[07:12:17] src/tree/updater_prune.cc:74: tree pruning end, 1 roots, 8 extra nodes, 0 pruned nodes, max_depth=3\u001b[0m\n",
      "\u001b[34m[07:12:17] src/tree/updater_prune.cc:74: tree pruning end, 1 roots, 6 extra nodes, 0 pruned nodes, max_depth=2\u001b[0m\n",
      "\u001b[34m[2]#011train-merror:0.02#011validation-merror:0.02\u001b[0m\n",
      "\u001b[34m[07:12:17] src/tree/updater_prune.cc:74: tree pruning end, 1 roots, 2 extra nodes, 0 pruned nodes, max_depth=1\u001b[0m\n",
      "\u001b[34m[07:12:17] src/tree/updater_prune.cc:74: tree pruning end, 1 roots, 8 extra nodes, 0 pruned nodes, max_depth=3\u001b[0m\n",
      "\u001b[34m[07:12:17] src/tree/updater_prune.cc:74: tree pruning end, 1 roots, 6 extra nodes, 0 pruned nodes, max_depth=2\u001b[0m\n",
      "\u001b[34m[3]#011train-merror:0.02#011validation-merror:0.02\u001b[0m\n",
      "\u001b[34m[07:12:17] src/tree/updater_prune.cc:74: tree pruning end, 1 roots, 2 extra nodes, 0 pruned nodes, max_depth=1\u001b[0m\n",
      "\u001b[34m[07:12:17] src/tree/updater_prune.cc:74: tree pruning end, 1 roots, 8 extra nodes, 0 pruned nodes, max_depth=3\u001b[0m\n",
      "\u001b[34m[07:12:17] src/tree/updater_prune.cc:74: tree pruning end, 1 roots, 6 extra nodes, 0 pruned nodes, max_depth=2\u001b[0m\n",
      "\u001b[34m[4]#011train-merror:0.02#011validation-merror:0.02\u001b[0m\n",
      "\u001b[34m[07:12:17] src/tree/updater_prune.cc:74: tree pruning end, 1 roots, 2 extra nodes, 0 pruned nodes, max_depth=1\u001b[0m\n",
      "\u001b[34m[07:12:17] src/tree/updater_prune.cc:74: tree pruning end, 1 roots, 6 extra nodes, 0 pruned nodes, max_depth=3\u001b[0m\n",
      "\u001b[34m[07:12:17] src/tree/updater_prune.cc:74: tree pruning end, 1 roots, 8 extra nodes, 0 pruned nodes, max_depth=3\u001b[0m\n",
      "\u001b[34m[5]#011train-merror:0.02#011validation-merror:0.04\u001b[0m\n",
      "\u001b[34m[07:12:17] src/tree/updater_prune.cc:74: tree pruning end, 1 roots, 2 extra nodes, 0 pruned nodes, max_depth=1\u001b[0m\n",
      "\u001b[34m[07:12:17] src/tree/updater_prune.cc:74: tree pruning end, 1 roots, 8 extra nodes, 0 pruned nodes, max_depth=3\u001b[0m\n",
      "\u001b[34m[07:12:17] src/tree/updater_prune.cc:74: tree pruning end, 1 roots, 8 extra nodes, 0 pruned nodes, max_depth=3\u001b[0m\n",
      "\u001b[34m[6]#011train-merror:0.02#011validation-merror:0.04\u001b[0m\n",
      "\u001b[34m[07:12:17] src/tree/updater_prune.cc:74: tree pruning end, 1 roots, 2 extra nodes, 0 pruned nodes, max_depth=1\u001b[0m\n",
      "\u001b[34m[07:12:17] src/tree/updater_prune.cc:74: tree pruning end, 1 roots, 8 extra nodes, 0 pruned nodes, max_depth=3\u001b[0m\n",
      "\u001b[34m[07:12:17] src/tree/updater_prune.cc:74: tree pruning end, 1 roots, 6 extra nodes, 0 pruned nodes, max_depth=2\u001b[0m\n",
      "\u001b[34m[7]#011train-merror:0.01#011validation-merror:0.06\u001b[0m\n",
      "\u001b[34m[07:12:17] src/tree/updater_prune.cc:74: tree pruning end, 1 roots, 2 extra nodes, 0 pruned nodes, max_depth=1\u001b[0m\n",
      "\u001b[34m[07:12:17] src/tree/updater_prune.cc:74: tree pruning end, 1 roots, 8 extra nodes, 0 pruned nodes, max_depth=3\u001b[0m\n",
      "\u001b[34m[07:12:17] src/tree/updater_prune.cc:74: tree pruning end, 1 roots, 6 extra nodes, 0 pruned nodes, max_depth=2\u001b[0m\n",
      "\u001b[34m[8]#011train-merror:0.01#011validation-merror:0.06\u001b[0m\n",
      "\u001b[34m[07:12:17] src/tree/updater_prune.cc:74: tree pruning end, 1 roots, 2 extra nodes, 0 pruned nodes, max_depth=1\u001b[0m\n",
      "\u001b[34m[07:12:17] src/tree/updater_prune.cc:74: tree pruning end, 1 roots, 10 extra nodes, 0 pruned nodes, max_depth=4\u001b[0m\n",
      "\u001b[34m[07:12:17] src/tree/updater_prune.cc:74: tree pruning end, 1 roots, 6 extra nodes, 0 pruned nodes, max_depth=2\u001b[0m\n",
      "\u001b[34m[9]#011train-merror:0.01#011validation-merror:0.06\u001b[0m\n",
      "\n",
      "2023-03-22 07:12:34 Uploading - Uploading generated training model\n",
      "2023-03-22 07:12:34 Completed - Training job completed\n",
      "Training seconds: 92\n",
      "Billable seconds: 92\n"
     ]
    },
    {
     "data": {
      "application/javascript": [
       "\n",
       "            setTimeout(function() {\n",
       "                var nbb_cell_id = 19;\n",
       "                var nbb_unformatted_code = \"xgb_model.fit({\\\"train\\\": train_ip, \\\"validation\\\": valid_ip}, wait=True, job_name=job_name)\";\n",
       "                var nbb_formatted_code = \"xgb_model.fit({\\\"train\\\": train_ip, \\\"validation\\\": valid_ip}, wait=True, job_name=job_name)\";\n",
       "                var nbb_cells = Jupyter.notebook.get_cells();\n",
       "                for (var i = 0; i < nbb_cells.length; ++i) {\n",
       "                    if (nbb_cells[i].input_prompt_number == nbb_cell_id) {\n",
       "                        if (nbb_cells[i].get_text() == nbb_unformatted_code) {\n",
       "                             nbb_cells[i].set_text(nbb_formatted_code);\n",
       "                        }\n",
       "                        break;\n",
       "                    }\n",
       "                }\n",
       "            }, 500);\n",
       "            "
      ],
      "text/plain": [
       "<IPython.core.display.Javascript object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "xgb_model.fit({\"train\": train_ip, \"validation\": valid_ip}, wait=True, job_name=job_name)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "                           PRE iris-xgboost-2023-02-03-06-03-15/\r\n",
      "                           PRE iris-xgboost-2023-03-10-06-20-53/\r\n",
      "                           PRE iris-xgboost-2023-03-10-17-15-32/\r\n",
      "                           PRE iris-xgboost-2023-03-10-17-25-29/\r\n",
      "                           PRE iris-xgboost2023-03-22-18-09-35/\r\n"
     ]
    },
    {
     "data": {
      "application/javascript": [
       "\n",
       "            setTimeout(function() {\n",
       "                var nbb_cell_id = 20;\n",
       "                var nbb_unformatted_code = \"!aws s3 ls {bucket}/{prefix}/model/\";\n",
       "                var nbb_formatted_code = \"!aws s3 ls {bucket}/{prefix}/model/\";\n",
       "                var nbb_cells = Jupyter.notebook.get_cells();\n",
       "                for (var i = 0; i < nbb_cells.length; ++i) {\n",
       "                    if (nbb_cells[i].input_prompt_number == nbb_cell_id) {\n",
       "                        if (nbb_cells[i].get_text() == nbb_unformatted_code) {\n",
       "                             nbb_cells[i].set_text(nbb_formatted_code);\n",
       "                        }\n",
       "                        break;\n",
       "                    }\n",
       "                }\n",
       "            }, 500);\n",
       "            "
      ],
      "text/plain": [
       "<IPython.core.display.Javascript object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "!aws s3 ls {bucket}/{prefix}/model/"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "                           PRE debug-output/\r\n",
      "                           PRE output/\r\n",
      "                           PRE profiler-output/\r\n"
     ]
    },
    {
     "data": {
      "application/javascript": [
       "\n",
       "            setTimeout(function() {\n",
       "                var nbb_cell_id = 21;\n",
       "                var nbb_unformatted_code = \"!aws s3 ls {bucket}/{prefix}/model/{job_name}/\";\n",
       "                var nbb_formatted_code = \"!aws s3 ls {bucket}/{prefix}/model/{job_name}/\";\n",
       "                var nbb_cells = Jupyter.notebook.get_cells();\n",
       "                for (var i = 0; i < nbb_cells.length; ++i) {\n",
       "                    if (nbb_cells[i].input_prompt_number == nbb_cell_id) {\n",
       "                        if (nbb_cells[i].get_text() == nbb_unformatted_code) {\n",
       "                             nbb_cells[i].set_text(nbb_formatted_code);\n",
       "                        }\n",
       "                        break;\n",
       "                    }\n",
       "                }\n",
       "            }, 500);\n",
       "            "
      ],
      "text/plain": [
       "<IPython.core.display.Javascript object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "!aws s3 ls {bucket}/{prefix}/model/{job_name}/"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.13"
  },
  "toc": {
   "base_numbering": 1,
   "nav_menu": {},
   "number_sections": true,
   "sideBar": true,
   "skip_h1_title": false,
   "title_cell": "Table of Contents",
   "title_sidebar": "Contents",
   "toc_cell": false,
   "toc_position": {},
   "toc_section_display": true,
   "toc_window_display": false
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}