experiments/cls_test_wt103_1_f.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Test of classifcation Fastai tokenization"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "%reload_ext autoreload\n",
    "%autoreload 2\n",
    "%matplotlib inline\n",
    "%cd .."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "from fastai import *\n",
    "from fastai.text import *"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "from ulmfit.train_clas import *"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Batch size: 70\n",
      "Max vocab: 60000\n",
      "Cache dir: data/imdb/models/f60k\n",
      "Model dir: data/imdb/models/f60k/lstm_None.m\n"
     ]
    }
   ],
   "source": [
    "exp = CLSHyperParams('data/imdb', qrnn=False,tokenizer='f', lang='en', cuda_id=0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "exp.pretrained_model = URLs.WT103_1\n",
    "exp.drop_mult=0.3"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Saving tokenized: cls.trn 25000, cls.val 25000\n",
      "Size of vocabulary: 60002\n",
      "First 20 words in vocab: ['xxunk', 'xxpad', 'xxmaj', 'the', '.', ',', 'and', 'a', 'of', 'to', 'is', 'it', 'in', 'i', 'this', 'that', '\"', \"'s\", '-', '\\n\\n']\n"
     ]
    }
   ],
   "source": [
    "data_clas, data_lm = exp.load_cls_data(bs=40,force=False, use_test_for_validation=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "40"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data_lm.train_dl.batch_size"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "true_wd:  False\n"
     ]
    }
   ],
   "source": [
    "learn = exp.create_lm_learner(data_lm)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "learn.true_wd=True\n",
    "learn.opt=None"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "LR Finder is complete, type {learner_name}.recorder.plot() to see the graph.\n"
     ]
    }
   ],
   "source": [
    "learn.lr_find()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAZgAAAEKCAYAAAAvlUMdAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4wLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvqOYd8AAAIABJREFUeJzt3Xd8VfX9+PHXOxMICSEkrLDCBpEZtoKCX+rE4rYu0JZatVKttfVnv7Zfq3XX8W2/RcVN60KpeyCKOFhho0DAsJJAEhISsuf798c9wQAJCXBP7sj7+XjcR8494573Jze57/s5n3FEVTHGGGO8LcTXARhjjAlOlmCMMca4whKMMcYYV1iCMcYY4wpLMMYYY1xhCcYYY4wrLMEYY4xxhSUYY4wxrrAEY4wxxhVhvg7AW+Lj47VXr16+DsMYYwLK6tWr96tqghuvHTQJplevXqSkpPg6DGOMCSgissut17ZLZMYYY1xhCcYYY4wrLMEYY4xxhSUYY4wxrrAEY4wxxhWWYIwxxrjCEowxxhhXWIIxxpgA9tbqdF5dudvXYdTLEowxxgSwBavTeXtNuq/DqJclGGOMCWB5xRXERUX4Oox6WYIxxpgAlltcQVxUpK/DqJclGGOMCVA1NcqBkgo6WA3GGGOMNxWUVlJdo3aJzBhjjHflFlcA0KGtJRhjjDFelOckmBZZgxGRnSKyUUTWichRN2sRj6dEZLuIbBCRkUdsjxGRDBH5u5txGmNMIMorLgf8N8E0xw3HzlTV/Q1sOwfo5zzGAv90ftb6C/Clu+EZY0xgOnSJzHqR1etC4GX1WA7EikgXABEZBXQCPvVlgMYY46/yijwJpn1UuI8jqZ/bCUaBT0VktYjMrmd7IrCnzvN0IFFEQoDHgN8d68VFZLaIpIhISk5OjteCNsaYQJBbXEF0ZBiRYaG+DqVebieYiao6Es+lsJtFZNIR26WeYxS4CfhQVffUs/3HHVWfUdVkVU1OSEjwTsTGGBMg8ooriPPTHmTgchuMqmY6P7NFZCEwBlhaZ5d0oHud592ATGA8cLqI3AS0BSJEpEhV/+BmvMYYE0j8eZoYcLEGIyJRIhJduwxMAzYdsdu7wLVOb7JxQIGq7lXVq1S1h6r2Au7A005jycUYY+rILfbfUfzgbg2mE7BQRGrP829V/VhEbgRQ1bnAh8C5wHagBJjlYjzGGBNU8orLOTUxxtdhNMi1BKOqacCwetbPrbOswM2NvM6LwIteDs8YYwKaqjqXyPyzizL4vpuyMcaYE1BYXkVltfr1JTJLMMYYE4Bqx8C0yEZ+Y4wx7qkdxe/P3ZQtwRhjTADKOzRNjCUYY4wxXuTvE12CJRhjjAlI/j7RJViCMcaYgJRXVEHr8FBaR/jnPGRgCcYYYwJSXnGF397JspYlGGOMCUD+Pk0MWIIxxpiA5O8TXYIlGGOMCUj+Pk0MWIIxxpiAlFtcbm0wxhhjvKukooqyyhq7RGaMMca7cgNgHjKwBGOMMQEnEKaJAUswxhgTcGoTTIuuwYjIThHZKCLrRCSlnu0iIk+JyHYR2SAiI531w0VkmYh856y/3M04jTEmkATCNDHg7i2Ta52pqvsb2HYO0M95jAX+6fwsAa5V1W0i0hVYLSKfqGp+M8RrjDF+7dBEl37ei6w5EsyxXAi87Nw6ebmIxIpIF1VNrd1BVTNFJBtIACzBGGNavNziCiLCQojy43nIwP02GAU+FZHVIjK7nu2JwJ46z9OddYeIyBggAvjBtSiNMSaA5BZ5pokREV+Hckxu12AmOjWQjsAiEdmiqkvrbK/vt6OHNop0AV4BrlPVmiN3dJLWbIAePXp4N3JjjPFTgTBNDLhcg1HVTOdnNrAQGHPELulA9zrPuwGZACISA3wA/FFVlzfw+s+oarKqJickJJxQjPklFTz52Ta+yyw4oeONMaa55bb0BCMiUSISXbsMTAM2HbHbu8C1Tm+ycUCBqu4VkQg8CellVX3TrRid2Pjfz7fx3vq9bp7GGGO8Jq+43O/HwIC7NZhOwNcish5YCXygqh+LyI0icqOzz4dAGrAdeBa4yVl/GTAJmOl0cV4nIsPdCLJd63BG94pj8eYsN17eGGO8Lq/I/ye6BBfbYFQ1DRhWz/q5dZYVuLmefeYD892K7UhTB3Xkvg82szu3hB4d2jTXaY0x5riVVVZTXFHt9xNdgo3kB+CsQZ0AWLzFajHGGP8WKKP4wRIMAL3io+iTEMXizdm+DsUYY47JEkwAOmtQJ1bsyKWwrNLXoRhjTINyA2SiS7AEc8jUQZ2orFaWpjY0q40xxvjeoWliLMEEjpE9YoltE269yYwxfq32XjD+PtElWII5JCw0hDMHdOSLrdlU12jjBxhjjA/kFVcQFiLEtPb1VJKNswRTx9RBHTlQUsma3Qd8HYoxxtQrr7iC9gEwDxlYgjnMpP4JhIUIn9llMmOMn8otrgiIBn6wBHOYmFbhjO0dZ92VjTF+K1AmugRLMEeZOrAT27OL2JVb7OtQjDHmKHnFFXRo6/8N/GAJ5iiHRvVbLcYY44dyCstJsAQTmHp0aENSfBTf/pDr61CMMeYwJRVVFJVXkRBtCSZgjekVx6qdedRYd2VjjB/ZX+gZA2MJJoCNSYqjoLSS1OxCX4dijDGH5BSVAZZgAtrY3nEArEjL83Ekxhjzo5xCzzQx1gYTwLq1b0NibGtW7rAEY4zxH4cSjNVgQER2ishG546UKfVsFxF5SkS2i8gGERlZZ9t1IrLNeVznZpz1GZMUx4odeXjuiWaMMb6XU1hOiATGRJfQPDWYM1V1uKom17PtHKCf85gN/BNAROKAPwFjgTHAn0SkfTPEesiYpDj2F5WzY7+NhzHG+IeconI6tI0kNMT/p4kB318iuxB4WT2WA7Ei0gX4CbBIVfNU9QCwCDi7OQMbm+S0w9hlMmOMnwikMTDgfoJR4FMRWS0is+vZngjsqfM83VnX0PpmkxQfRXzbSGuHMcb4jZzC8oBpfwFwe77niaqaKSIdgUUiskVVl9bZXl89T4+x/jBO0poN0KNHD2/EW/e1GZsUZwnGGOM3cgrL6dcp2tdhNJmrNRhVzXR+ZgML8bSn1JUOdK/zvBuQeYz1R77+M6qarKrJCQkJ3gwd8LTDZOSXkn6gxOuvbYwxx0NVySkKrBqMawlGRKJEJLp2GZgGbDpit3eBa53eZOOAAlXdC3wCTBOR9k7j/jRnXbOy8TDGGH9RUFpJZbUGVBuMm5fIOgELnZvihAH/VtWPReRGAFWdC3wInAtsB0qAWc62PBH5C7DKea17VbXZP+X7d4ymXetwVu7I4+JR3Zr79MYYc0igjYEBFxOMqqYBw+pZP7fOsgI3N3D888DzbsXXFCEhwuhecazcaTUYY4xvBWKC8XU3Zb83NimOHfuLyT5Y5utQjDEtWE6RJZigc6gdxnqTGWN8yGowQWhwlxiiIkJZlmb3hzHG+E5OYTmRYSFER7o9usR7LME0Iiw0hEn9E/j0uyyq7f4wxhgfqR1k6XScCgiWYJrggmFd2V9UznKrxRhjfCTQxsCAJZgmmTKwI1ERoby3/qixnsYY0yxyCsuJD6AxMGAJpklahYcy7ZTOfLRpHxVVNb4OxxjTAgXaPGRgCabJLhjWhYLSSr7enuPrUIwxLUxldQ15JRUBNYofLME02Wl9E2jXOpx319llMmNM88orrkA1sLoogyWYJosIC+GcIZ1Z9H0WpRXVvg7HGNOCBOIYGLAEc1ymD+tKcUU1X2zN9nUoxpgWxBJMCzC2dwfi20baZTJjTLM6lGCsDSZ4hYYI5w/twudbsyksq/R1OMaYFiIQ5yEDSzDH7YJhXaioqmHR91m+DsUY00LkFJYT3SqMVuGhvg7luFiCOU4je7QnMbY1c7/8gYz8Ul+HY4xpAQJxDAxYgjluIsJ9Px1CZn4Z5z31FUuswd8Y47KcwvKAa38BSzAn5MyBHXnv16fROaYVs15cxd8WpdpEmMYY1wTiPGTQDAlGREJFZK2IvF/Ptp4islhENojIEhHpVmfbwyLynYhsFpGnxM+mEE2Kj2LhTRO5ZGQ3nlq8jd+8vs7XIRljgpRdImvYHGBzA9seBV5W1aHAvcADACIyAZgIDAWGAKOBye6HenxaR4TyyKXDmDWxF+9vyCS/pMLXIRljgkxJRRVF5VWWYI7k1EjOA+Y1sMtgYLGz/AVwobOsQCsgAogEwgG/7bZ1/tCuqMLX2/f7OhRjTJDZX+j54mptMEd7ArgTaGgK4vXAxc7yDCBaRDqo6jI8CWev8/hEVY+qBYnIbBFJEZGUnBzfTUI5rFs7YlqFsTTVJsI0xnhXTlEZEHhjYMDFBCMi5wPZqrr6GLvdAUwWkbV4LoFlAFUi0hcYBHQDEoEpIjLpyINV9RlVTVbV5ISEBO8XoonCQkOY2Deer7btR9Ua+40x3hOo08SAuzWYicB0EdkJvIYnScyvu4OqZqrqRao6ArjbWVeApzazXFWLVLUI+AgY52KsJ21S/wT2FpSxPbvI16EYY4KIJZh6qOpdqtpNVXsBVwCfq+rVdfcRkXgRqY3hLuB5Z3k3nppNmIiE46ndNNRRwC9M6u+pQX1pl8mMMV6UU1hOiECHqCBNMCLSR0QineUzRORWEYk9kROKyL0iMt15egawVURSgU7A/c76BcAPwEY87TTrVfW9Ezlfc0mMbU2fhCiWbrOGfmOM9+QUlRMXFUloiF+N1GiSsCbu9xaQ7LSNPAe8C/wbOLcpB6vqEmCJs3xPnfUL8CSTI/evBn7ZxNj8xun9Enh15W7KKqsDbs4gY4x/CtQxMND0S2Q1qlqFp23kCVW9DejiXliBaXL/BMqrali1M8/XoRhjgkRLSDCVInIlcB1QOyI/3J2QAtfY3nFEhIZYd2VjjNcE6jxk0PQEMwsYD9yvqjtEJAmY38gxLU6biDCSe7XnK2uHMcZ4QU2NklNUTseYIE4wqvq9qt6qqq+KSHsgWlUfdDm2gDSpfwJb9hWSdbDM16G0SNU1amORTNDILiynslrpGtva16GckCY18ovIEmC6s/86IEdEvlTV212MLSBN6pfAgx9tYWlqDpcmd2/SMUXlVfz53e84WOq5S6YCoSLcOrUfg7vGuBhtcFmelsutr66lb8e2PHbZMLq0C8x/SmNqZeSXANAtQBNMUy+RtVPVg8BFwAuqOgo4y72wAtfAztHEt408rstkzy5NY8HqdHbnlbA7r4Q9eSV8tS2HP727ybVv4zmF5SxPyw2Kb/uqytwvf+CqeStoFR7Kuj35nPPkV3zy3T5fh2bMSUk/4LmpYWL7wEwwTe2mHCYiXYDLcEbcm/qFhAiT+sXzxdZsqqprCAs9dg4/UFzBc1/v4OxTOjP3mlGH1r+yfBf//Z9NLEnN4cwBHb0SW1llNYs3Z/PWmnS+TM2hukaZM7Uft/1X/0aPPVhWye2vr6d7XGsuGdWNU7q280pMJ6ugtJI73lzPou+zOO/ULjx0yVByCsu59dW1/PKV1fxsbA/++7zBtI7w327jReVVRIaFEN7I34ppeWrvmpsYoDWYpiaYe4FPgG9UdZWI9Aa2uRdWYPvJkM68vTaDr7bvbzQ5zF36A8UVVdw+7fAP+cuTu/Ps0jQe+Xgrk/slEHLEICtV5Vi3yNmxv5gPN+4lp7Cc/UXl5BSWs3nvQQ6WVdEpJpLZk3qTVVDGk4u3ER4q3DKlX4OvVVOj3P76Or7YmkOoCC98s5OBnaO5ZFQ3rhjTg7aRTf0z8q7Vu/K47fX1ZOaX8t/nD+b6ib0QEdpGhvHWrybw2KKtPP1lGst+yOWhi4cyJinOJ3Ee6d31mXz2fdah2mpucQVd27XiuZmjGdTFLomaH2XmlxLbJpwoH/2PnawmRa2qbwJv1nmexo+zIJsjnDmgI7Ftwnl7TcYxE0z2wTJe+nYnPx2eSP9O0YdtiwgL4bb/6sdtr6/ng417uWBY10PbcgrLmfnCSjrFtOKpK0cc9QH/feZBfjZvOfkllURHhhEfHUlC20imndKZ6cO6MrFvPKEhcugunI9+mkpYaAg3Tu5Tb5x//2I7n23O5s8XDOanIxJ5b30mC1anc98Hm1n2Qy7PzRx9or+qE1JWWc3jn6XyzNI0EmNb89rscST3Ojx5RISFcNc5g5jcL4Hfv72By55exrXje3Ln2QN9lhBVlacWb+fxz1Lp0q4VfRLaMu2UziTGtmL+8t1cOncZ/3fVyEPTDhmTcaCUrgHcltjURv5uwP/imcBSga+BOaqa7mJsASsiLIQLhnbljZQ9FJZVEt2q/iFD//hiO5XVnstU9Zk+LJG5S9L426JUzhnSmbDQEPYXlXPVvOXsyi1hy75Cfvbscp6fOZp4p5/85r0HuWreclqHh/KfOybSKz6qwThDQ4RHLh1GVY3y4EdbCAsRfn5678P2+WJLNo9/lsqMEYlcN8FTQ7hmfC+uGd+LuV/+wIMfbWHx5iymDup0gr+t47Mpo4Db31hHalYRV47pzt3nDT5mwpjQN55PfjOJRz7Zyovf7mTx5mweuXQoE/rEey2mvOIK2rcJP2aNsqZGuff973nx251cPLIbD1186mGXTy8e1Y1ZL6xi1our+OuMIVw+uofX4jOBKyO/lJ4dGv4f9ndNvej7Ap7pYbrimT7/PWedacBFIxMpr6rho431NzSnHyjh3yt3c1lytwaTQGiIcMdPBrBjfzELVqeTW1TO1fNWsDuvhBdmjuaZa0aRmlXIJf/8lj15JWzZd/BQQ/drs8cdM7nUPcffLhvGead24b4PNnPOk1/x3Nc72F9Uzq7cYua8tpaBnWP464xTj/oAvX5iEn0Sorj3/e8pq6w+/l/Scdqy7yAX/d+35JdU8sKs0Txw0dAm1UbaRITxpwtO4c1fjicyPISZz69ieVquV2L65Lt9JN+3iD+8tZGamvo7TFRW1/DbN9fz4rc7uX5iEo9cMvSotrku7Vrz5o3jmdg3nt+/tZGHP97S4OuZlkFVyThQGrDtLwDSlF5EIrJOVYc3ts6XkpOTNSUlxddhHKKqTH3sSxKiI3n9l+OP2v77BRtYuDaDJb8745h93FWVi/75LfsKymjXOpwd+4t5fuZoJvb1fANfvesA17+4ioiwEKprlIjQkCYnl7oqq2t4bdUeFqTsYX16AWEhQrvW4VTVKO/dcho9OrSp97ivt+3n6udW8Nv/6s+vG6iJeYOqcvVzK/gu8yCf3T75UI3teB0oruDSp5eRVVDGGzeOP6k2j43pBVz69LdEtwonp7CcS0Z146GLhx42KWHWwTJ+t2ADS1NzuGNaf24+s+8xazqV1TXc8853vLpyN2cOSOCJy0fQro1NmtESFZRUMuzeT/njeYOOurLgTSKyWlWT3XjtptZg9ovI1SIS6jyuBrzzFTBIiQgXjUxkxY489uSVHLZt675CFqxJ56pxPRodQCUi/O4nA9hbUEba/mLmXZd8KLkAjOrZngU3jicsRAgPFV49geQCEB4awjXjevLOLafx6W2TuOG0JDq0jeDvPxvRYHIBOK1fPOee2pl/LNlO+oGSBvc7WZ9tzuab7bncdlb/E04uAO2jInjp+jG0iQxl5gsrTzjmzPxSbnhpFR2iIvnw1tP5zVn9WLA6nTsXbDg02PO1lbs5629fsiItl7/OOJVbpvQ7ZnIBz/vw1xlD+MtPh/D19v1M/8fXbNl38IRiNIEt3RkD0xJqMD2Av+OZLkaBb4FbVXW3u+E1nb/VYAD25JVw+sNfHPbtvqCkkgv/8TVF5VV8NGdSkyexe/7rHQxJbNdgT6ii8ipqVIlpoL3HTRn5pZz12JecMSCBf149qvEDjlN5VTU/eXwp4aEhfDTn9Ea7fjfFln0HuXTuMhKiI3nrxgm0j4po8rFF5VVcOncZ6XklvHXThEMdNJ78bBuPf5bK+UO7kFdcwbc/5DI2KY4HLx5K0gkk/dW78vjV/DUUllXx8CVDD+voYYLfp9/tY/Yrq3nn5okM635Cd0dpEjdrME3tRbYbz0j+ukH9BnjCjaCCRfe4NoxNimPh2gxumdKX6hrlllfXkJFfyqu/GHdcM6Ref1rSMbf7qmcUeL5h3TKlL498spWnFm+jbWQYxeVVFFdU07NDGy4amUhk2ImPQ3n5213szC3hpevHeCW5AAzsHMO8a5O55vmVXDVvBX+7fBgDOzd+uWxfQRl3vb2B1KxCXpg5+rDef3PO6ocI/G1RKtGRYfx1xqlcMbr7UV3Mm2pUzzje//Vp3PSvNfz61bWEhwpnD7FJzFuKQ2NgAnSQJTSxBlPvgSK7VdVvurr4Yw0G4I1Ve7jzrQ0svGkC72/Yy3Nf7+Dhi4dy2eimTSMTKMqrqjnnya9Iyyk+tC48VKisVhJjWzNnaj8uGpl43Akit6icMx5ZQnKv9rwwa4y3w+bzLVn89o31FJZVccPpScyZ2o82ET8m69KKatbuPsCXqTl8mZrDln2FANw/YwhXje1Z72t+tS2H/p2i6RTTyisxllVWc+Wzy9m89yBv/nICp3bzj0Guxl33vf8981fsYvO9Zzd6afVkuFmDOZkEs0dVG/2UFJFQIAXIUNXzj9jWE89tkhOAPODq2q7PzmW5eUB3PJflzlXVnQ2dx18TTGFZJcn3fUaPuDZsyy5i5oRe/Hn6Kb4OyxXF5VXkl1YSFRFKm4gwwkOFr7bt57FFqazfk0+vDm34/dkDOefUpn8Lv3vhRl5btYdPfjOJvh3buhL3geIKHvhoM2+kpJMY25obTktiZ24xa3fns3nvQapqlPBQYXSvOCb3T2DKwI70O2LckttyCsv56T++oaqmhnduPo3O7byTvIz/uulfq9myr5DPf3uGq+fx1wTTpBqMiNwOJAMx9SSYN4H3VfUlEZkCzFLVa5xtS/DcHmCRiLTFc9OzBltk/TXBAPz61bW8tz6TiX078NIs713mCRSqyuLN2Tz66Va27CtkztR+/Oasxhu8v0zNYdYLK7l2fPMk5ZU78rh74Ua2ZRcRFRHKsO6xjOgRy8ge7Rnbu4NPL0OCZ4zTJf/8lt4JbXnjl+P9evobc/Iu/PvXxLQO55Ubxrp6Hp+1wYhIIZ7aw1GbgEYvDDoDNM8D7gfqm3l5MHCbs/wF8B/nuMFAmKouAlDVosbO5c9uPrMPYSHCPecPbnHJBTw94c4a3InJAxK46+2NPLl4G/sKyrh/xpAGfx/vrMvgt2+sp3+naG47q/G50rxhTFIcH845nYwDpXSPa+N390Af1CWGp64cwc9fTuE3r6/lscuG+zzpGfdk5JcG/NRBx/y0U9VoVY2p5xGtqk35y34CuBOoaWD7en6ccmYGEC0iHYD+QL6IvC0ia0XkEedSW0Aa2DmGxy8fflw9lYJReGgIj1wylFun9OX1lD384uUUSiqqjtrvua93MOe1dYzq2Z43bhzfrONAwkND6BUf5XfJpdbUQZ3443mD+eS7LCY8sJiHPt5i9x4KQmWV1ewvqgjoLsrQ9Mkuj5uInA9kq+pqETmjgd3uAP4uIjOBpUAGUOXEdTowAtgNvA7MBJ474hyzgdkAPXr4TX8Dcwwiwu3TBtCpXSv++z+bOO+prxnTK47eCVH0TmjLqp15PLM0jXOGdObxy4fTKjxgv1e45obTkhjZI5Z5X+3g6S9/YN5XaUw7pTOhIuwrKGPvwVKKyqr48/RTuHB4oq/DNScgGHqQgYsJBs+8ZdNF5FygFRAjIvNV9eraHVQ1E889ZnDaWS5W1QIRSQfWOpNqIiL/AcZxRIJR1WeAZ8DTBuNiWYyXXTW2J13atWLukjQWb8ni9ZSKQ9uuHteD/5k+xG9rEf5gRI/2/OOq9uzOLeH5b3bw7vpMoluF0TmmFaN6tGdbdhF/eGsjQxLb0SfBnc4Rxj2ZAT5Nf60TbuQ/rpN4ajB31NPIHw/kqWqNiNwPVKvqPc7lsDXAWaqaIyIvACmq+o+GzuHPjfymcQUllaTtL6KiqoYxSXGudstsCbIOlnH2E0vp0q41C2+ecFLjkEzze23lbv7w9ka+/v2ZdGvf8Ewa3uAPU8V4jYjcKyK1gzbPALaKSCrQCU9nAFS1Gs/ls8UishFPp4JnmztW03zatQlnhNNby5LLyesU04pHLx3G93sP8uBHW3wdjjlOGfmlhAheG0vlK83SBUVVlwBLnOV76qxfACxo4JhFwNBmCM+YoDR1UCdmTujFC9/s5LS+8c12SwVz8jIOlNI5plXA3+U0sKM3xhzTXecOZHCXGO54cz37Cqy3WaBIzy8N+AZ+sARjTFCLDAvlf382grLKGv707iZfh2OaKDM/sO8DU8sSjDFBrk9CW26Z0pdPvsviq205vg7HNKK6RtlXUGY1GGNMYPj56Un07NCGP7/7HRVVDY17Nv4g62AZVTXa6L2iAoElGGNagMiwUO45fzA/5BTz0rc7fR2OOYaMIBkDA5ZgjGkxpg7qxJkDEnhy8TayC63B31/VDrLsZpfIjDGB5J4LTqGiqoaHPtrq61BMA9IPeBKMXSIzxgSUpPgobjg9ibfWpLN61wFfh2PqkZFfSlxUxGE3vgtUlmCMaWFuObMvXdq14qZ/rWbn/uLGDzDNKuNAKV1jA3sEfy1LMMa0MFGRYbw4awyV1cqVzy5nd26D9/EzPpARJGNgwBKMMS3SgM7RzL9hLKWV1Vz57HLSD1iS8Qeq6gyydHeCy+ZiCcaYFmpw1xjm3zCWwrJKrnx2+aHeS8Z3cosrKKmoDooeZGAJxpgWbUhiO165YSz5xZVc/dwK8ksqGj/IuCY1qxCAfp2C4x4+lmCMaeGGdY/luZmjSc8rZfbLqymrrPZ1SC3WtqwiAPp3ivZxJN5hCcYYw5ikOB67bBgrd+bxuwUbqKk5/EaEB8sqSc0qPGq98a7UrEJiWoXRMTrS16F4ReB3tDbGeMUFw7qSfqCUhz7eQmJsa/5wzkB+yCnipW93smB1OiUV1XSIimBi33hO7xfPgM7RbN1XyMaMAjakF1BQWskjlwwluVecr4sSsFKzCunfKTpobrrneoJxbn+cAmTUc8vknsDzQAKQB1ytqul1tscAm4GFqnqL27Ea09LdOLk36QdKmPvlDyxtI6sKAAAWOklEQVRPy2XdnnwiQkO4YFhXxiS1Z0VaHku37efd9ZmHjmkbGcaQxBhyi2uY9eIqXv3FOIYktvNhKQKTqpKaVcR5Q7v4OhSvaY4azBw8SSKmnm2PAi+r6ksiMgV4ALimzva/AF+6H6IxBkBE+J/pp5BTWM66PfncdlZ/fja2BwnOJZvLR/dAVdmyr5C0nGIGdokmqUMUISFCRn4pl/7zW657fiVv3DiePgnB0VDdXHIKyykoraR/x+D5vbnaBiMi3YDzgHkN7DIYWOwsfwFcWOfYUUAn4FM3YzTGHC4sNISnrxnFiv83lTln9TuUXGqJCIO6xHDe0C70SWhLSIjnck5ibGvm/3wsInD1vBU2tuY4pQZZAz+438j/BHAn0NANKNYDFzvLM4BoEekgIiHAY8DvXI7PGFMPETmhdoDeCW15+fqxFJdXcfW8FRwotm7PTbX1UBdlSzCNEpHzgWxVXX2M3e4AJovIWmAykAFUATcBH6rqnkbOMVtEUkQkJSfH7tRnjD8Y3DWGF2aNZmduCS8t2+nrcALGtqxC4qIiiG8b4etQvMbNGsxEYLqI7AReA6aIyPy6O6hqpqpepKojgLuddQXAeOAW59hHgWtF5MEjT6Cqz6hqsqomJyQkuFgUY8zxGNUzjsn9E3h15W4qq+0Omk2RmlVIv45tg6YHGbiYYFT1LlXtpqq9gCuAz1X16rr7iEi8czkM4C48PcpQ1atUtYdz7B14OgL8wa1YjTHed824nmQdLGfR91m+DsXvqSrbsoqCqv0FfDDQUkTuFZHpztMzgK0ikoqnQf/+5o7HGOOOMwd2JDG2Na8s2+XrUPze3oIyCsur6N85uBJMswy0VNUlwBJn+Z466xcACxo59kXgRdeCM8a4IjREuGpcDx7+eCvbsgqDqvHa22rnIAumLspgU8UYY1x0eXJ3IkJDmL/cajHHcijBBFkStgRjjHFNh7aRnDe0C2+tyaCovMrX4fit1Kwi4ttG0j4qeHqQgSUYY4zLrhnfk6LyKv6zNsPXofitbVmFDOgcXJfHwBKMMcZlI7rHckrXGF5ZtgtVJSO/lPc3ZPLAR5vZlFHg6/B8rqbGMwdZv47BdXkMbDZlY4zLRIRrx/fk929tJPm+z8itM7r/3XWZfDTndGLbBNeloeORkV9KaWV10LW/gCUYY0wzmD4skUXfZxPTKozhPWIZ0b09FdU1XPHMMu5csIGnrxkVVAMMj0dtA38wXiKzBGOMcV3riFDmXZd81Po7fzKQ+z/czPzlu7hmfK/mD8wP1M5B1jcIL5FZG4wxxmduOC2Jyf0T+MsHm9m896Cvw/GJbVlFdI5pRbvW4b4OxesswRhjfCYkRHjssmG0ax3Or19dS0lFy+vKnJpVSL9OwXd5DCzBGGN8LL5tJI9fNpwfcor43YINVLWgyTGra5Tt2UUMCMIGfrAEY4zxA6f1i+cPZw/kgw17ue2N9S0myezKLaa8qiYoe5CBNfIbY/zELyf3oUbhoY+3APD4ZcMICw3u78ApOw8AMLxHrI8jcYclGGOM3/jVGX0QgQc/ahlJZnlaLnFREfQLskkua1mCMcb4lRsn9wE8SaZtZBgPXHSqjyNyz4odeYxNigvaMUDB+9XAGBOwbpzch+snJvHaqt18nxmc3Zf35JWQkV/K2KQ4X4fiGkswxhi/NGdqP6Ijw3j0062+DsUVy9NyARjXp4OPI3GPJRhjjF9q1yacG8/ow+dbslm5I8/X4Xjdih15xLYJp38QjuCv5XqCEZFQEVkrIu/Xs62niCwWkQ0iskREujnrh4vIMhH5ztl2udtxGmP8z6wJSXSMjuThj7egqr4Ox6tW7MhlTK84QkKCs/0FmqcGMwfY3MC2R4GXVXUocC/wgLO+BLhWVU8BzgaeEJHg7MdnjGlQ64hQbp3aj5RdB/h8S7avw/GajPxS9uSVMq538F4eA5cTjFMjOQ+Y18Aug4HFzvIXwIUAqpqqqtuc5UwgG0hwM1ZjjH+6fHR3enZow8Mfb6W6JjhqMSuc9pexvYO3gR/cr8E8AdwJNDQsdz1wsbM8A4gWkcNSuoiMASKAH448WERmi0iKiKTk5OR4L2pjjN8IDw3ht9MGsDWrkHfXB8ddMVek5RHTKoyBnWN8HYqrXEswInI+kK2qq4+x2x3AZBFZC0wGMoBDs92JSBfgFWCWqh6VpFT1GVVNVtXkhASr4BgTrM4/tQundI3h0U9Sg2JCzBU7chmT1IHQIG5/AXdrMBOB6SKyE3gNmCIi8+vuoKqZqnqRqo4A7nbWFQCISAzwAfBHVV3uYpzGGD8XEiLcc/5gMvJL+fvn230dzknZV1DGztwSxgX55TFwMcGo6l2q2k1VewFXAJ+r6tV19xGReBGpjeEu4HlnfQSwEE8HgDfditEYEzjG9u7ARSMTefarNLZnF/o6nBO2YofT/pIU3A384INxMCJyr4hMd56eAWwVkVSgE3C/s/4yYBIwU0TWOY/hzR2rMca//L9zB9EmIow//mdTwHZbXp6WR3RkGIO7Bnf7CzTTXGSqugRY4izfU2f9AmBBPfvPB+Yfud4Y07LFt43kzrMHcPfCTSxcm8FFI7v5OqTjtiItl9FJcUHf/gI2kt8YE2CuHN2D4d1juf+DzRSUVPo6nOOSfbCMtP3FQT3/WF2WYIwxASUkRLjvp0M4UFLBn9/7jgPFFb4OqckWO4NFJ/SJ93EkzcOm6zfGBJwhie34xem9eXppGu+syyC5ZxxTBnXknCGd6dkhytfhNei1lbsZ0CmaIYnB3/4CVoMxxgSoP5wzkHdunsgtZ/alqLyKBz/awrTHl7Irt9jXodVrU0YB69MLuGJM96C9/8uRLMEYYwKSiDCseyy3TxvAh3NOZ/FvJ1OjynNf7/B1aPV6bdVuIsNCmDEi0dehNBtLMMaYoNAnoS0/HZ7IGyl7/K5dpqSiinfWZnLeqV2IbRPh63CajSUYY0zQ+MWk3pRV1jB/+S5fh3KY9zfspbC8iivG9PB1KM3KEowxJmj07xTNGQMSeGnZTsoqq30dziGvrtxNn4QoRvdq7+tQmpUlGGNMUJk9qTf7iypYuNY/Zl7esu8ga3fnc+WYHi2mcb+WJRhjTFAZ37sDQxJjeParNGr84P4xr63cQ0RoSEDOOnCyLMEYY4KKiPCL03uTllPs87tgllVW8/aadM4e0pm4qJbTuF/LEowxJuice2oXEmNb88xXaT6N46NNezlYVsUVY7r7NA5fsQRjjAk64aEhzJrYi5U78li7+4DP4nh7TQbd41ozrgVMzV8fSzDGmKB0xZgexLQK4+kvfVOLyTpYxjfb9zNjeCIhLWDm5PpYgjHGBKW2kWFcM74nn3y/j7ScomY//zvrMqhRmNECG/drWYIxxgStmROSCA8N4VkftMW8vSaD4d1jSYr338k33eZ6ghGRUBFZKyLv17Otp4gsFpENIrJERLrV2XadiGxzHte5HacxJvgkREdy6ahuvLU6g+yDZc123u8zD7JlXyEXjWw5847VpzlqMHOAzQ1sexR4WVWHAvcCDwCISBzwJ2AsMAb4k4i0rCGwxhivmD2pN1U1Nbzw7c5mO+fCtemEhQjnD+3abOf0R64mGKdGch4wr4FdBgOLneUvgAud5Z8Ai1Q1T1UPAIuAs92M1RgTnHp2iOKcU7swf/kuCsvcvwNmdY3yzrpMzhjQsUWOfanL7RrME8CdQE0D29cDFzvLM4BoEekAJAJ76uyX7qwzxpjjduOkPhSWVfHvFbtdP9c32/eTXVje4i+PgYsJRkTOB7JVdfUxdrsDmCwia4HJQAZQBdTXp++oOR9EZLaIpIhISk5OjjfCNsYEoVO7tWNi3w489/UOSivcnQRz4doMoluFMWVgR1fPEwjcrMFMBKaLyE7gNWCKiMyvu4OqZqrqRao6ArjbWVeAp8ZSd+hrNyDzyBOo6jOqmqyqyQkJCS4VwxgTDG46oy/ZheWM/etn3PX2Rlak5Xp9rrLi8io+3rSP84d2oVV4qFdfOxC5lmBU9S5V7aaqvYArgM9V9eq6+4hIvIjUxnAX8Lyz/AkwTUTaO43705x1xhhzQib2jeffPx/L1EGd+M/aDC5/ZjmnP/wFq3d5b6T/u+szKa2sZsaIljv2pa5mHwcjIveKyHTn6RnAVhFJBToB9wOoah7wF2CV87jXWWeMMSdsQt94Hr98OCl/PIsnLh8OwJ0L1lNR1VAzcdOt3nWA/3nvO4Z3jyW5p3V6BRBV309n7Q3JycmakpLi6zCMMQFk8eYsbngphbvPHcQvJvU+4dfZnl3IJXOXEds6nAW/mkB820gvRukuEVmtqsluvLaN5DfGtFhTB3ViysCOPLl42wkPxNxXUMZ1z68iLCSEl68fG1DJxW2WYIwxLdo95w+moqqGBz/actzHFpRWct3zKykoreTFWaPp0aGNCxEGLkswxpgWrVd8FL+YlMTbazNYtbNpTb2qyrvrMznvqa9I21/E3KtHMSSxncuRBp4wXwdgjDG+dvOZfXl7TQb3vPMd828Yw4odeXy1LYdvtufSJiKU0/rGc1q/eMYmdeC7zALu+2Az6/bkM7BzNI/eMJZxvVvm/V4aY438xhgDfLBhLzf/e82h59GRYYzr04GSiipW7ThARXUN4aFCZbXSMTqSO34ygItHdiM0wO/14mYjv9VgjDEGOPfUztw6tR8CTOofz7BusYSFeloRSiuqWbkzj2+276d9mwium9CTNhH28dkYq8EYY0wLZt2UjTHGBBxLMMYYY1xhCcYYY4wrLMEYY4xxhSUYY4wxrrAEY4wxxhWWYIwxxrjCEowxxhhXBM1ASxHJAXYdsbodUHCc6xpbjgf2n2CY9Z37ePZpSnmaqyyNxdrYPsdbliOf1y7XXWfvTdNibWwfe298+xlwrP3cKEuUqrpzz3lVDdoH8MzxrmtsGUjxZjzHs09TytNcZTnZ8hxvWY5Rhrrr7L2x98av35umlMWb743bf2eNPYL9Etl7J7CuKcvejOd49mlKeZqrLE19nYb2Od6yHPn8vQb2OVH23hx7vb03zfcZcKz9/KksjQqaS2TNRURS1KV5e5pbMJUFgqs8wVQWCK7yWFmaLthrMG54xtcBeFEwlQWCqzzBVBYIrvJYWZrIajDGGGNcYTUYY4wxrmjRCUZEnheRbBHZdALHjhKRjSKyXUSeEhGps+3XIrJVRL4TkYe9G3WD8Xi9LCLyZxHJEJF1zuNc70feYEyuvDfO9jtEREUk3nsRHzMeN96bv4jIBud9+VREuno/8nrjcaMsj4jIFqc8C0Uk1vuRNxiTG+W51PnfrxER19tqTqYMDbzedSKyzXlcV2f9Mf+v6uVmFzV/fwCTgJHAphM4diUwHhDgI+AcZ/2ZwGdApPO8YwCX5c/AHcHy3jjbugOf4BkzFR+oZQFi6uxzKzA3gMsyDQhzlh8CHgrkvzNgEDAAWAIk+2sZnPh6HbEuDkhzfrZ3ltsfq7zHerToGoyqLgXy6q4TkT4i8rGIrBaRr0Rk4JHHiUgXPP/gy9Tzm38Z+Kmz+VfAg6pa7pwj291SeLhUFp9xsTyPA3cCzdb46EZZVPVgnV2jaKbyuFSWT1W1ytl1OdDN3VL8yKXybFbVrc0Rv3O+EypDA34CLFLVPFU9ACwCzj7Rz4kWnWAa8Azwa1UdBdwB/F89+yQC6XWepzvrAPoDp4vIChH5UkRGuxrtsZ1sWQBucS5dPC8i7d0LtUlOqjwiMh3IUNX1bgfaBCf93ojI/SKyB7gKuMfFWBvjjb+zWtfj+XbsS94sj680pQz1SQT21HleW64TKm9YE0/aIohIW2AC8Gady4uR9e1az7rab5BheKqW44DRwBsi0tvJ+s3GS2X5J/AX5/lfgMfwfAA0u5Mtj4i0Ae7GcznGp7z03qCqdwN3i8hdwC3An7wcaqO8VRbnte4GqoB/eTPG4+HN8vjKscogIrOAOc66vsCHIlIB7FDVGTRcrhMqryWYw4UA+ao6vO5KEQkFVjtP38XzwVu3Gt8NyHSW04G3nYSyUkRq8Mz3k+Nm4PU46bKoalad454F3ncz4EacbHn6AEnAeuefrhuwRkTGqOo+l2M/kjf+zur6N/ABPkgweKksTmPy+cDU5v4ydgRvvze+UG8ZAFT1BeAFABFZAsxU1Z11dkkHzqjzvBuetpp0TqS8bjdA+fsD6EWdxjHgW+BSZ1mAYQ0ctwpPLaW2wetcZ/2NwL3Ocn881U0J0LJ0qbPPbcBrgfzeHLHPTpqpkd+l96ZfnX1+DSwI4LKcDXwPJDTn35fbf2c0UyP/iZaBhhv5d+C5CtPeWY5rSnnrjcsXb6i/PIBXgb1AJZ4MfQOeb7kfA+udP/p7Gjg2GdgE/AD8nR8HrUYA851ta4ApAVyWV4CNwAY839q6NEdZ3CrPEfvspPl6kbnx3rzlrN+AZ16pxAAuy3Y8X8TWOY9m6RHnYnlmOK9VDmQBn/hjGagnwTjrr3fek+3ArMbKe6yHjeQ3xhjjCutFZowxxhWWYIwxxrjCEowxxhhXWIIxxhjjCkswxhhjXGEJxgQ1ESlq5vPNE5HBXnqtavHMlrxJRN5rbJZhEYkVkZu8cW5jvMG6KZugJiJFqtrWi68Xpj9OzOiqurGLyEtAqqref4z9ewHvq+qQ5ojPmMZYDca0OCKSICJvicgq5zHRWT9GRL4VkbXOzwHO+pki8qaIvAd8KiJniMgSEVkgnvuY/Kv23hjO+mRnuciZkHK9iCwXkU7O+j7O81Uicm8Ta1nL+HHSzrYislhE1ojn/hwXOvs8CPRxaj2POPv+zjnPBhH5Hy/+Go1plCUY0xI9CTyuqqOBi4F5zvotwCRVHYFnduK/1jlmPHCdqk5xno8AfgMMBnoDE+s5TxSwXFWHAUuBX9Q5/5PO+Rudz8mZB2sqntkUAMqAGao6Es/9hx5zEtwfgB9Udbiq/k5EpgH9gDHAcGCUiExq7HzGeItNdmlaorOAwXVmmo0RkWigHfCSiPTDM1NseJ1jFqlq3XturFTVdAARWYdnLqivjzhPBT9OELoa+C9neTw/3kvj38CjDcTZus5rr8Zzbw7wzAX1VydZ1OCp2XSq5/hpzmOt87wtnoSztIHzGeNVlmBMSxQCjFfV0rorReR/gS9UdYbTnrGkzubiI16jvM5yNfX/L1Xqj42cDe1zLKWqOlxE2uFJVDcDT+G5/0sCMEpVK0VkJ9CqnuMFeEBVnz7O8xrjFXaJzLREn+K5fwoAIlI7rXk7IMNZnuni+ZfjuTQHcEVjO6tqAZ7bIt8hIuF44sx2ksuZQE9n10Igus6hnwDXO/cHQUQSRaSjl8pgTKMswZhg10ZE0us8bsfzYZ3sNHx/j+cWCwAPAw+IyDdAqIsx/Qa4XURWAl2AgsYOUNW1eGbGvQLPDbmSRSQFT21mi7NPLvCN0635EVX9FM8luGUishFYwOEJyBhXWTdlY5qZc3fNUlVVEbkCuFJVL2zsOGMCjbXBGNP8RgF/d3p+5eOj21Ab4zarwRhjjHGFtcEYY4xxhSUYY4wxrrAEY4wxxhWWYIwxxrjCEowxxhhXWIIxxhjjiv8Pjaxti3ClkuwAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "learn.recorder.plot(skip_end=15)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Total time: 23:05\n",
      "epoch  train_loss  valid_loss  accuracy\n",
      "1      4.216088    4.007746    0.300710  (23:05)\n",
      "\n"
     ]
    }
   ],
   "source": [
    "learn.fit_one_cycle(1, 1e-02, moms=(0.8,0.7))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "learn.save('fit_head')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "learn.load('fit_head');"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Total time: 8:42:10\n",
      "epoch  train_loss  valid_loss  accuracy\n",
      "1      3.969496    3.845486    0.315829  (26:05)\n",
      "2      3.918823    3.763834    0.326209  (26:03)\n",
      "3      3.839340    3.699592    0.335176  (26:08)\n",
      "4      3.806484    3.647501    0.342129  (26:06)\n",
      "5      3.774351    3.600486    0.347226  (26:12)\n",
      "6      3.743252    3.558260    0.352164  (26:09)\n",
      "7      3.699686    3.523352    0.356172  (26:09)\n",
      "8      3.713620    3.493051    0.360080  (26:11)\n",
      "9      3.655959    3.463499    0.363502  (26:11)\n",
      "10     3.639268    3.436016    0.366489  (26:08)\n",
      "11     3.612611    3.406506    0.370597  (26:11)\n",
      "12     3.583289    3.374530    0.374582  (26:07)\n",
      "13     3.538116    3.351603    0.378152  (26:08)\n",
      "14     3.525463    3.321655    0.382012  (26:06)\n",
      "15     3.490068    3.292532    0.385994  (26:08)\n",
      "16     3.455298    3.272834    0.388984  (26:11)\n",
      "17     3.408465    3.253918    0.391583  (26:05)\n",
      "18     3.420946    3.241025    0.393331  (25:56)\n",
      "19     3.382529    3.235213    0.394229  (25:57)\n",
      "20     3.367180    3.233525    0.394359  (25:49)\n",
      "\n"
     ]
    }
   ],
   "source": [
    "learn.unfreeze()\n",
    "learn.fit_one_cycle(20, 1e-3, moms=(0.8,0.7))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [],
   "source": [
    "learn.save(\"afteroom\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "learn.load(\"afteroom\");"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[3.2334335, tensor(0.3944)]"
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "learn.validate()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "# odl: [3.6938994, tensor(0.3338)]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [],
   "source": [
    "learn.save_encoder(\"enc_best\")\n",
    "learn.save(\"lm_best\", with_opt=False)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Classification"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Validation = 100% of test"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "exp.drop_mult=0.5\n",
    "learn=exp.create_cls_learner(data_clas)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "learn.true_wd=True"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "learn.load_encoder('enc_best')\n",
    "learn.freeze()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Total time: 04:03\n",
      "epoch  train_loss  valid_loss  accuracy\n",
      "1      0.286372    0.176776    0.933840  (04:03)\n",
      "\n"
     ]
    }
   ],
   "source": [
    "learn.fit_one_cycle(1, 2e-2, moms=(0.8,0.7))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Total time: 04:21\n",
      "epoch  train_loss  valid_loss  accuracy\n",
      "1      0.235155    0.160959    0.940560  (04:21)\n",
      "\n"
     ]
    }
   ],
   "source": [
    "learn.freeze_to(-2)\n",
    "learn.fit_one_cycle(1, slice(1e-2/(2.6**4),1e-2), moms=(0.8,0.7))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "learn.save(\"2\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Total time: 05:38\n",
      "epoch  train_loss  valid_loss  accuracy\n",
      "1      0.213980    0.148502    0.946840  (05:38)\n",
      "\n"
     ]
    }
   ],
   "source": [
    "learn.freeze_to(-3)\n",
    "learn.fit_one_cycle(1, slice(5e-3/(2.6**4),5e-3), moms=(0.8,0.7))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "learn.save(\"3\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Total time: 13:45\n",
      "epoch  train_loss  valid_loss  accuracy\n",
      "1      0.198522    0.153674    0.947800  (06:52)\n",
      "2      0.169954    0.157253    0.947320  (06:52)\n",
      "\n"
     ]
    }
   ],
   "source": [
    "learn.unfreeze()\n",
    "learn.fit_one_cycle(2, slice(1e-3/(2.6**4),1e-3), moms=(0.8,0.7))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "learn.save(\"all\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Validation = 10% of train"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Tokenized data loaded, lm.trn 90000, lm.val 10000\n",
      "Tokenized data loaded, cls.trn 22500, cls.val 2500\n",
      "Size of vocabulary: 60002\n",
      "First 20 words in vocab: ['xxunk', 'xxpad', 'xxmaj', 'the', '.', ',', 'and', 'a', 'of', 'to', 'is', 'it', 'in', 'i', 'this', 'that', '\"', \"'s\", '-', '\\n\\n']\n"
     ]
    }
   ],
   "source": [
    "data_clas, data_lm = exp.load_cls_data(bs=40,force=False, use_test_for_validation=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "exp.drop_mult=0.5\n",
    "learn=exp.create_cls_learner(data_clas)\n",
    "learn.true_wd=True"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "learn.load_encoder('enc_best')\n",
    "learn.freeze()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Total time: 02:43\n",
      "epoch  train_loss  valid_loss  accuracy\n",
      "1      0.288307    1.597378    0.575200  (02:43)\n",
      "\n"
     ]
    }
   ],
   "source": [
    "learn.fit_one_cycle(1, 2e-2, moms=(0.8,0.7))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Total time: 03:08\n",
      "epoch  train_loss  valid_loss  accuracy\n",
      "1      0.246644    0.435174    0.779200  (03:08)\n",
      "\n"
     ]
    }
   ],
   "source": [
    "learn.freeze_to(-2)\n",
    "learn.fit_one_cycle(1, slice(1e-2/(2.6**4),1e-2), moms=(0.8,0.7))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Total time: 04:19\n",
      "epoch  train_loss  valid_loss  accuracy\n",
      "1      0.202743    0.212411    0.923200  (04:19)\n",
      "\n"
     ]
    }
   ],
   "source": [
    "learn.freeze_to(-3)\n",
    "learn.fit_one_cycle(1, slice(5e-3/(2.6**4),5e-3), moms=(0.8,0.7))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Total time: 10:51\n",
      "epoch  train_loss  valid_loss  accuracy\n",
      "1      0.189055    0.162822    0.944000  (05:25)\n",
      "2      0.168473    0.165473    0.941600  (05:25)\n",
      "\n"
     ]
    }
   ],
   "source": [
    "learn.unfreeze()\n",
    "learn.fit_one_cycle(2, slice(1e-3/(2.6**4),1e-3), moms=(0.8,0.7))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [],
   "source": [
    "learn.save(\"valid\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Running models on testset\n",
    "There is a small bug that does not let us use test_ds so we load the test set in to valid_ds and run validate() on that data set"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Tokenized data loaded, lm.trn 90000, lm.val 10000\n",
      "Tokenized data loaded, cls.trn 25000, cls.val 25000\n",
      "Size of vocabulary: 60002\n",
      "First 20 words in vocab: ['xxunk', 'xxpad', 'xxmaj', 'the', '.', ',', 'and', 'a', 'of', 'to', 'is', 'it', 'in', 'i', 'this', 'that', '\"', \"'s\", '-', '\\n\\n']\n"
     ]
    }
   ],
   "source": [
    "data_clas, data_lm = exp.load_cls_data(bs=40,force=False, use_test_for_validation=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [],
   "source": [
    "learn=exp.create_cls_learner(data_clas)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [],
   "source": [
    "learn.load('valid');"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[0.15947564, tensor(0.9483)]"
      ]
     },
     "execution_count": 29,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "learn.validate()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [],
   "source": [
    "learn.load(\"all\");"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[0.15725298, tensor(0.9473)]"
      ]
     },
     "execution_count": 35,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "learn.validate()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "fastai-dev",
   "language": "python",
   "name": "fastai-dev"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.8"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}