05.Learning rates and learning rate schedulers.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "It helps to avoid local optimas when using smaller learning rates. However, it often takes longer to converge. What can help shorten the training time is using a warm-up period. In this period, we can use a bigger learning rate for the first few epochs. After a certain number of epochs, we can decrease the learning rate. \n",
    "<br><br>\n",
    "It's even possible to decrease the learning rate after each step, but this is not recommended, because you might be better off using a different optimizer instead (for example, if you want to use decay, you can specify this in as a hyperparameter). In theory, when the learning rate is too big during the warm-up period, it can be the case that you won't be able to reach the global optima at all."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import math\n",
    "\n",
    "from keras.models import Sequential\n",
    "from keras.layers import Dense, Dropout, Activation, Flatten\n",
    "from keras.layers import Conv2D, MaxPooling2D\n",
    "from keras.optimizers import SGD\n",
    "\n",
    "from keras.callbacks import EarlyStopping, TensorBoard, ModelCheckpoint, LearningRateScheduler, Callback\n",
    "from keras import backend as K"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "n_classes = 5"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_datagen = ImageDataGenerator(rescale=1./255,\n",
    "                                   shear_range=0.2,\n",
    "                                   zoom_range=0.2,\n",
    "                                   width_shift_range=0.1,\n",
    "                                   height_shift_range=0.1,\n",
    "                                   horizontal_flip=True,\n",
    "                                   vertical_flip=False,\n",
    "                                   validation_split=0.25)\n",
    "\n",
    "test_datagen = ImageDataGenerator(rescale=1./255)\n",
    "\n",
    "training_set = train_datagen.flow_from_directory('data',\n",
    "                                                target_size = (150,150),\n",
    "                                                 batch_size = batch_size,\n",
    "                                                 class_mode = 'categorical',\n",
    "                                                 subset = \"training\")\n",
    "\n",
    "validation_set = train_datagen.flow_from_directory('data',\n",
    "                                            target_size = (150,150),\n",
    "                                            batch_size = batch_size,\n",
    "                                            class_mode = 'categorical',\n",
    "                                            subset = \"validation\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Other than a custom callback function, Keras also provides a convenient LearningRateScheduler and ReduceLROnPlateau callback function. <br><br>With these callbacks, you can implement an epoch-dependent learning rate scheme or reduce the learning rate if a monitored loss or metric reaches a plateau."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "learning_rate_schedule = {0: '0.1', 10: '0.01', 25: '0.0025'}\n",
    "\n",
    "class get_learning_rate(Callback):\n",
    "    def on_epoch_end(self, epoch, logs={}):\n",
    "        optimizer = self.model.optimizer\n",
    "        if epoch in learning_rate_schedule:\n",
    "            K.set_value(optimizer.lr, learning_rate_schedule[epoch])\n",
    "        lr = K.eval(optimizer.lr)\n",
    "        print('\\nlr: {:.4f}'.format(lr))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "callbacks =[EarlyStopping(monitor='val_acc', patience=5, verbose=2),\n",
    "            ModelCheckpoint('checkpoints/{epoch:02d}.h5', \n",
    "            save_best_only=True),\n",
    "            TensorBoard('~/notebooks/logs-lrscheduler',\n",
    "            write_graph=True, write_grads=True, \n",
    "            write_images=True, embeddings_freq=0, \n",
    "            embeddings_layer_names=None,\n",
    "            embeddings_metadata=None),\n",
    "            get_learning_rate()\n",
    "            ]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "model = Sequential()\n",
    "model.add(Conv2D(32, (3, 3), padding='same', input_shape = (150, 150,3)))\n",
    "model.add(Activation('relu'))\n",
    "model.add(Conv2D(32, (3, 3)))\n",
    "model.add(Activation('relu'))\n",
    "model.add(MaxPooling2D(pool_size=(2, 2)))\n",
    "model.add(Dropout(0.25))\n",
    "\n",
    "model.add(Conv2D(64, (3, 3), padding='same'))\n",
    "model.add(Activation('relu'))\n",
    "model.add(Conv2D(64, (3, 3)))\n",
    "model.add(Activation('relu'))\n",
    "model.add(MaxPooling2D(pool_size=(2, 2)))\n",
    "model.add(Dropout(0.25))\n",
    "\n",
    "model.add(Flatten())\n",
    "model.add(Dense(512))\n",
    "model.add(Activation('relu'))\n",
    "model.add(Dropout(0.5))\n",
    "model.add(Dense(n_classes))\n",
    "model.add(Activation('softmax'))\n",
    "model.summary()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "optimizer = SGD()\n",
    "model.compile(loss='categorical_crossentropy', optimizer=optimizer, metrics=['accuracy'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "n_epochs = 20\n",
    "batch_size = 128\n",
    "\n",
    "history = model.fit(training_set, epochs=n_epochs, batch_size=batch_size, \n",
    "          validation_data = validation_set,\n",
    "          verbose = 1, callbacks=callbacks)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}