courses/machine_learning/deepdive/05_artandscience/b_hyperparam.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "deletable": true,
    "editable": true,
    "id": "4f3CKqFUqL2-",
    "slideshow": {
     "slide_type": "slide"
    }
   },
   "source": [
    "# Hyperparameter tuning with Cloud AI Platform"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true
   },
   "source": [
    "**Learning Objectives:**\n",
    "  * Improve the accuracy of a model by hyperparameter tuning"
   ]
  },
  {
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
 "!sudo chown -R jupyter:jupyter /home/jupyter/training-data-analyst"
   ]
  },
  {

"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
 "# Ensure the right version of Tensorflow is installed.\n",
 "!pip freeze | grep tensorflow==2.1"
 ]
},
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "import os\n",
    "PROJECT = 'cloud-training-demos' # REPLACE WITH YOUR PROJECT ID\n",
    "BUCKET = 'cloud-training-demos-ml' # REPLACE WITH YOUR BUCKET NAME\n",
    "REGION = 'us-central1' # REPLACE WITH YOUR BUCKET REGION e.g. us-central1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "# for bash\n",
    "os.environ['PROJECT'] = PROJECT\n",
    "os.environ['BUCKET'] = BUCKET\n",
    "os.environ['REGION'] = REGION\n",
    "os.environ['TFVERSION'] = '2.1'  # Tensorflow version"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "%%bash\n",
    "gcloud config set project $PROJECT\n",
    "gcloud config set compute/region $REGION"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "deletable": true,
    "editable": true,
    "id": "6TjLjL9IU80G"
   },
   "source": [
    "## Create command-line program\n",
    "\n",
    "In order to submit to Cloud AI Platform, we need to create a distributed training program. Let's convert our housing example to fit that paradigm, using the Estimators API."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true,
    "deletable": true,
    "editable": true,
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "%%bash\n",
    "rm -rf house_prediction_module\n",
    "mkdir house_prediction_module\n",
    "mkdir house_prediction_module/trainer\n",
    "touch house_prediction_module/trainer/__init__.py"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "%%writefile house_prediction_module/trainer/task.py\n",
    "import argparse\n",
    "import os\n",
    "import json\n",
    "import shutil\n",
    "\n",
    "from . import model\n",
    "    \n",
    "if __name__ == '__main__' and \"get_ipython\" not in dir():\n",
    "  parser = argparse.ArgumentParser()\n",
    "  parser.add_argument(\n",
    "      '--learning_rate',\n",
    "      type = float, \n",
    "      default = 0.01\n",
    "  )\n",
    "  parser.add_argument(\n",
    "      '--batch_size',\n",
    "      type = int, \n",
    "      default = 30\n",
    "  )\n",
    "  parser.add_argument(\n",
    "      '--output_dir',\n",
    "      help = 'GCS location to write checkpoints and export models.',\n",
    "      required = True\n",
    "  )\n",
    "  parser.add_argument(\n",
    "      '--job-dir',\n",
    "      help = 'this model ignores this field, but it is required by gcloud',\n",
    "      default = 'junk'\n",
    "  )\n",
    "  args = parser.parse_args()\n",
    "  arguments = args.__dict__\n",
    "  \n",
    "  # Unused args provided by service\n",
    "  arguments.pop('job_dir', None)\n",
    "  arguments.pop('job-dir', None)\n",
    "  \n",
    "  # Append trial_id to path if we are doing hptuning\n",
    "  # This code can be removed if you are not using hyperparameter tuning\n",
    "  arguments['output_dir'] = os.path.join(\n",
    "      arguments['output_dir'],\n",
    "      json.loads(\n",
    "          os.environ.get('TF_CONFIG', '{}')\n",
    "      ).get('task', {}).get('trial', '')\n",
    "  )\n",
    "  \n",
    "  # Run the training\n",
    "  shutil.rmtree(arguments['output_dir'], ignore_errors=True) # start fresh each time\n",
    "  \n",
    "  # Pass the command line arguments to our model's train_and_evaluate function\n",
    "  model.train_and_evaluate(arguments)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true,
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "%%writefile house_prediction_module/trainer/model.py\n",
    "\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "import tensorflow as tf\n",
    "\n",
    "tf.compat.v1.logging.set_verbosity(tf.compat.v1.logging.INFO)\n",
    "\n",
    "# Read dataset and split into train and eval\n",
    "df = pd.read_csv(\"https://storage.googleapis.com/ml_universities/california_housing_train.csv\", sep = \",\")\n",
    "df['num_rooms'] = df['total_rooms'] / df['households']\n",
    "np.random.seed(seed = 1) #makes split reproducible\n",
    "msk = np.random.rand(len(df)) < 0.8\n",
    "traindf = df[msk]\n",
    "evaldf = df[~msk]\n",
    "\n",
    "# Train and eval input functions\n",
    "SCALE = 100000\n",
    "\n",
    "def train_input_fn(df, batch_size):\n",
    "  return tf.compat.v1.estimator.inputs.pandas_input_fn(x = traindf[[\"num_rooms\"]],\n",
    "                                             y = traindf[\"median_house_value\"] / SCALE,  # note the scaling\n",
    "                                             num_epochs = None,\n",
    "                                             batch_size = batch_size, # note the batch size\n",
    "                                             shuffle = True)\n",
    "\n",
    "def eval_input_fn(df, batch_size):\n",
    "  return tf.compat.v1.estimator.inputs.pandas_input_fn(x = evaldf[[\"num_rooms\"]],\n",
    "                                             y = evaldf[\"median_house_value\"] / SCALE,  # note the scaling\n",
    "                                             num_epochs = 1,\n",
    "                                             batch_size = batch_size,\n",
    "                                             shuffle = False)\n",
    "\n",
    "# Define feature columns\n",
    "features = [tf.feature_column.numeric_column('num_rooms')]\n",
    "\n",
    "def train_and_evaluate(args):\n",
    "  # Compute appropriate number of steps\n",
    "  num_steps = (len(traindf) / args['batch_size']) / args['learning_rate']  # if learning_rate=0.01, hundred epochs\n",
    "\n",
    "  # Create custom optimizer\n",
    "  myopt = tf.compat.v1.train.FtrlOptimizer(learning_rate = args['learning_rate']) # note the learning rate\n",
    "\n",
    "  # Create rest of the estimator as usual\n",
    "  estimator = tf.compat.v1.estimator.LinearRegressor(model_dir = args['output_dir'], \n",
    "                                           feature_columns = features, \n",
    "                                           optimizer = myopt)\n",
    "  #Add rmse evaluation metric\n",
    "  def rmse(labels, predictions):\n",
    "    pred_values = tf.cast(predictions['predictions'], tf.float64)\n",
    "    return {'rmse': tf.compat.v1.metrics.root_mean_squared_error(labels * SCALE, pred_values * SCALE)}\n",
    "  estimator = tf.compat.v1.estimator.add_metrics(estimator, rmse)\n",
    "\n",
    "  train_spec = tf.estimator.TrainSpec(input_fn = train_input_fn(df = traindf, batch_size = args['batch_size']),\n",
    "                                      max_steps = num_steps)\n",
    "  eval_spec = tf.estimator.EvalSpec(input_fn = eval_input_fn(df = evaldf, batch_size = len(evaldf)),\n",
    "                                    steps = None)\n",
    "  tf.estimator.train_and_evaluate(estimator, train_spec, eval_spec)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true,
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "%%bash\n",
    "rm -rf house_trained\n",
    "export PYTHONPATH=${PYTHONPATH}:${PWD}/house_prediction_module\n",
    "gcloud ai-platform local train \\\n",
    "    --module-name=trainer.task \\\n",
    "    --job-dir=house_trained \\\n",
    "    --package-path=$(pwd)/trainer \\\n",
    "    -- \\\n",
    "    --batch_size=30 \\\n",
    "    --learning_rate=0.02 \\\n",
    "    --output_dir=house_trained"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true
   },
   "source": [
    "# Create hyperparam.yaml"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true,
    "deletable": true,
    "editable": true,
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "%%writefile hyperparam.yaml\n",
    "trainingInput:\n",
    "  hyperparameters:\n",
    "    goal: MINIMIZE\n",
    "    maxTrials: 5\n",
    "    maxParallelTrials: 1\n",
    "    hyperparameterMetricTag: rmse\n",
    "    params:\n",
    "    - parameterName: batch_size\n",
    "      type: INTEGER\n",
    "      minValue: 8\n",
    "      maxValue: 64\n",
    "      scaleType: UNIT_LINEAR_SCALE\n",
    "    - parameterName: learning_rate\n",
    "      type: DOUBLE\n",
    "      minValue: 0.01\n",
    "      maxValue: 0.1\n",
    "      scaleType: UNIT_LOG_SCALE"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true,
    "deletable": true,
    "editable": true,
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "%%bash\n",
    "OUTDIR=gs://${BUCKET}/house_trained   # CHANGE bucket name appropriately\n",
    "gsutil rm -rf $OUTDIR\n",
    "export PYTHONPATH=${PYTHONPATH}:${PWD}/house_prediction_module\n",
    "gcloud ai-platform jobs submit training house_$(date -u +%y%m%d_%H%M%S) \\\n",
    "   --config=hyperparam.yaml \\\n",
    "   --module-name=trainer.task \\\n",
    "   --package-path=$(pwd)/house_prediction_module/trainer \\\n",
    "   --job-dir=$OUTDIR \\\n",
    "   --runtime-version 2.1 \\\n",
    "   --python-version 3.5 \\\n",
    "   --\\\n",
    "   --output_dir=$OUTDIR \\"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true,
    "deletable": true,
    "editable": true,
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "!gcloud ai-platform jobs describe house_180912_195904 # CHANGE jobId appropriately"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true
   },
   "source": [
    "## Challenge exercise\n",
    "Add a few engineered features to the housing model, and use hyperparameter tuning to choose which set of features the model uses.\n",
    "\n",
    "<p>\n",
    "Copyright 2020 Google Inc. Licensed under the Apache License, Version 2.0 (the \"License\"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License"
   ]
  }
 ],
 "metadata": {
  "colab": {
   "default_view": {},
   "name": "first_steps_with_tensor_flow.ipynb",
   "provenance": [],
   "version": "0.3.2",
   "views": {}
  },
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}