Episode_2_Classifying_with_Vectors.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Classifying Text with Vectors\n",
    "\n",
    "* Last episode we converted text to vectors and used those vectors to find similar documents. \n",
    "* This episode we are going to use those vectors for classification.\n",
    "* We are also going to use machine learning to make the vectors align with our classification goal."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "from typing import List, Tuple\n",
    "import os.path\n",
    "from pathlib import Path\n",
    "import random\n",
    "\n",
    "import torch\n",
    "import torch.nn.functional as F\n",
    "from transformers import BertModel, BertTokenizer, BertConfig"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Install TensorBoard\n",
    "\n",
    "We are going to use it to graph our results.\n",
    "\n",
    "`pip install tensorboard`"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "from torch.utils.tensorboard import SummaryWriter"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Let's get the data\n",
    "\n",
    "* We are going to take my tweets and classify whether they are about movies or not.\n",
    "* Since most tweets aren't about movies, we have to down sample. Imbalanced classes are a whole topic that we can cover another time."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Data directory:  /Users/jmugan/temp\n",
      "Downsampled dataset: 45 positive tweets out of 101\n"
     ]
    }
   ],
   "source": [
    "home = str(Path.home())\n",
    "data_dir = os.path.join(home, 'temp')\n",
    "print(\"Data directory: \", data_dir)\n",
    "\n",
    "all_tweets: List[str] = []\n",
    "with open('jmugan_tweets.txt', 'r') as f:\n",
    "    for tweet in f:\n",
    "        tweet = tweet.strip()  # remove newline\n",
    "        all_tweets.append(tweet)\n",
    "\n",
    "def is_movie_tweet(tweet: str) -> bool:\n",
    "    return 'movie' in tweet.lower()\n",
    "\n",
    "def down_sample(all_tweets: List[str]) -> List[str]:\n",
    "    down_sampled = []\n",
    "    num_pos = 0\n",
    "    for tweet in all_tweets:\n",
    "        if is_movie_tweet(tweet):\n",
    "            num_pos += 1\n",
    "            down_sampled.append(tweet)\n",
    "        elif random.random() > .95:  # let in 80% of others\n",
    "            down_sampled.append(tweet)\n",
    "    print(f\"Downsampled dataset: {num_pos} positive tweets out of {len(down_sampled)}\")\n",
    "    return down_sampled\n",
    "\n",
    "# Normally, we would break the data into train, valid, and test, \n",
    "# but this is just a quick video\n",
    "training_tweets = down_sample(all_tweets)\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# We need to `Dataset` to represent our data set"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "from torch.utils.data import Dataset\n",
    "\n",
    "# https://pytorch.org/docs/stable/data.html\n",
    "# You don't have to load the whole data into memory\n",
    "class TweetDataset(torch.utils.data.Dataset):\n",
    "\n",
    "    def __init__(self, tweets: List[str]):\n",
    "        self.tweets = tweets\n",
    "\n",
    "    def __len__(self):\n",
    "        return len(self.tweets)\n",
    "\n",
    "    def __getitem__(self, index: int):\n",
    "        return self.tweets[index]\n",
    "\n",
    "    #def __iter__(self):\n",
    "    #    # you could be pulling these from a file instead so the whole thing\n",
    "    #    # doesn't have to sit in memory, see torch.utils.data.IterableDataset\n",
    "    #    for tweet in self.tweets:\n",
    "    #        yield tweet\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# We need a `DataLoader` to batch and preprocess our data and pass it to the learner"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "# recall from last time\n",
    "def get_tokens(text: str,\n",
    "               tokenizer: BertTokenizer,\n",
    "               config: BertConfig) -> List[str]:\n",
    "    tokens = tokenizer.tokenize(text)\n",
    "    # make sure it isn't too long\n",
    "    max_length = config.max_position_embeddings\n",
    "    tokens = tokens[:max_length-1] # Will add special begin token\n",
    "    # cls token to hold vector https://huggingface.co/transformers/main_classes/tokenizer.html\n",
    "    tokens = [tokenizer.cls_token] + tokens\n",
    "    return tokens\n",
    "\n",
    "\n",
    "\n",
    "def get_labels(batch: List[str]) -> List[int]:\n",
    "    \"\"\"\n",
    "    Normally you have the labeled data from somewhere, maybe somebody hand labeled it.\n",
    "    Here, we will just use a simple-stupid function\n",
    "    \"\"\"\n",
    "    labels: List[int] = []\n",
    "    for tweet in batch:\n",
    "        if is_movie_tweet(tweet):\n",
    "            labels.append(1)\n",
    "        else:\n",
    "            labels.append(0)\n",
    "    return labels\n",
    "\n",
    "\n",
    "# We need a function that puts masks on because the tweets have different lengths\n",
    "def preprocess_batch(batch: List[str]\n",
    "    ) -> Tuple[torch.Tensor, torch.Tensor, torch.Tensor, List[str]]:\n",
    "    \"\"\"\n",
    "    We get a list of batch-size in and we have to convert it to ids and make a mask\n",
    "    \"\"\"\n",
    "\n",
    "    # First tokenize\n",
    "    tokenized_tweets: List[List[str]] = [get_tokens(tweet, tokenizer, config) \n",
    "                                         for tweet in batch]\n",
    "\n",
    "    # find the max length\n",
    "    lengths = [len(tokenized_tweet) for tokenized_tweet in tokenized_tweets]\n",
    "    max_length = max(lengths)\n",
    "\n",
    "    # get batch size\n",
    "    batch_size = len(batch)\n",
    "\n",
    "    # let's make it tensors\n",
    "    input_data = torch.zeros(batch_size, max_length, dtype=torch.long)\n",
    "    mask_data = torch.zeros(batch_size, max_length, dtype=torch.long)\n",
    "\n",
    "    for i, tokenized_tweet in enumerate(tokenized_tweets):\n",
    "        token_ids = tokenizer.convert_tokens_to_ids(tokenized_tweet)\n",
    "        tensor = torch.tensor(token_ids)\n",
    "        input_data[i,:len(token_ids)] = tensor\n",
    "        mask_data[i,:len(token_ids)] = 1\n",
    "\n",
    "    # get labels\n",
    "    labels = get_labels(batch)\n",
    "    labels_tensor = torch.tensor(labels, dtype=torch.long)\n",
    "\n",
    "    return input_data, mask_data, labels_tensor, batch\n",
    "\n",
    "\n",
    "from torch.utils.data import DataLoader\n",
    "\n",
    "BATCH_SIZE = 20\n",
    "train_data_loader = DataLoader(\n",
    "    TweetDataset(training_tweets),\n",
    "    batch_size = BATCH_SIZE,\n",
    "    shuffle = True,\n",
    "    collate_fn=preprocess_batch\n",
    ")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Now let's define our classifier as a PyTorch module\n",
    "* It takes a Bert model as the encoder and declares a matrix of size `hidden_size x 2` to map the Bert vector down to two dimensions (tweet about movies or not)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "class Classifier(torch.nn.Module):\n",
    "    def __init__(self, encoder: BertModel):\n",
    "        \"\"\"\n",
    "        The init function of a module specifies the model parameters\n",
    "        \"\"\"\n",
    "        super().__init__()\n",
    "        self.encoder = encoder\n",
    "        # We have two classes, about movies and not about movies\n",
    "        self.classes = torch.nn.Linear(encoder.config.hidden_size, 2)\n",
    "\n",
    "    def forward(self, input_data: torch.Tensor,\n",
    "                mask_data: torch.Tensor) -> Tuple[torch.Tensor, torch.Tensor]:\n",
    "        \"\"\"\n",
    "        Forward is called when you call the model instance\n",
    "        \"\"\"\n",
    "\n",
    "        # The mask is so it doesn't pay attention to any tokens that are just \n",
    "        # filling up space. You need this because the input tensor is of shape \n",
    "        # batch_size x max_len where max_len is the longest tweet in that batch.\n",
    "        _last_hidden_state, pooler_output = self.encoder(input_data, \n",
    "                                                         attention_mask = mask_data)\n",
    "\n",
    "        # shape batch_size x encoder.config.hidden_size\n",
    "        vectors = pooler_output\n",
    "\n",
    "        # shape batch_size x 2\n",
    "        logits = self.classes(vectors)\n",
    "\n",
    "        # shape batch_size x 2; do softmax and log to pass \n",
    "        # into negative log likelihood later\n",
    "        log_softmax = F.log_softmax(logits, dim=1)\n",
    "\n",
    "        # We will use the vectors in the next video\n",
    "        return log_softmax, vectors\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Let's pull up our Bert model like we did before"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "model_name = 'bert-base-uncased'\n",
    "# Need to use the same tokenizer that was used to train the model so that it breaks up words\n",
    "# into tokens the same way.\n",
    "tokenizer = BertTokenizer.from_pretrained(model_name)\n",
    "\n",
    "# This model is huge!!!!!!!!\n",
    "model = BertModel.from_pretrained(model_name)\n",
    "\n",
    "# What do I use this for?\n",
    "config = BertConfig.from_pretrained(model_name)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# We can now instantiate our classifier"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Classifier(\n",
       "  (encoder): BertModel(\n",
       "    (embeddings): BertEmbeddings(\n",
       "      (word_embeddings): Embedding(30522, 768, padding_idx=0)\n",
       "      (position_embeddings): Embedding(512, 768)\n",
       "      (token_type_embeddings): Embedding(2, 768)\n",
       "      (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
       "      (dropout): Dropout(p=0.1, inplace=False)\n",
       "    )\n",
       "    (encoder): BertEncoder(\n",
       "      (layer): ModuleList(\n",
       "        (0): BertLayer(\n",
       "          (attention): BertAttention(\n",
       "            (self): BertSelfAttention(\n",
       "              (query): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (key): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (value): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (dropout): Dropout(p=0.1, inplace=False)\n",
       "            )\n",
       "            (output): BertSelfOutput(\n",
       "              (dense): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
       "              (dropout): Dropout(p=0.1, inplace=False)\n",
       "            )\n",
       "          )\n",
       "          (intermediate): BertIntermediate(\n",
       "            (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
       "          )\n",
       "          (output): BertOutput(\n",
       "            (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
       "            (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
       "            (dropout): Dropout(p=0.1, inplace=False)\n",
       "          )\n",
       "        )\n",
       "        (1): BertLayer(\n",
       "          (attention): BertAttention(\n",
       "            (self): BertSelfAttention(\n",
       "              (query): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (key): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (value): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (dropout): Dropout(p=0.1, inplace=False)\n",
       "            )\n",
       "            (output): BertSelfOutput(\n",
       "              (dense): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
       "              (dropout): Dropout(p=0.1, inplace=False)\n",
       "            )\n",
       "          )\n",
       "          (intermediate): BertIntermediate(\n",
       "            (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
       "          )\n",
       "          (output): BertOutput(\n",
       "            (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
       "            (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
       "            (dropout): Dropout(p=0.1, inplace=False)\n",
       "          )\n",
       "        )\n",
       "        (2): BertLayer(\n",
       "          (attention): BertAttention(\n",
       "            (self): BertSelfAttention(\n",
       "              (query): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (key): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (value): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (dropout): Dropout(p=0.1, inplace=False)\n",
       "            )\n",
       "            (output): BertSelfOutput(\n",
       "              (dense): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
       "              (dropout): Dropout(p=0.1, inplace=False)\n",
       "            )\n",
       "          )\n",
       "          (intermediate): BertIntermediate(\n",
       "            (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
       "          )\n",
       "          (output): BertOutput(\n",
       "            (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
       "            (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
       "            (dropout): Dropout(p=0.1, inplace=False)\n",
       "          )\n",
       "        )\n",
       "        (3): BertLayer(\n",
       "          (attention): BertAttention(\n",
       "            (self): BertSelfAttention(\n",
       "              (query): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (key): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (value): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (dropout): Dropout(p=0.1, inplace=False)\n",
       "            )\n",
       "            (output): BertSelfOutput(\n",
       "              (dense): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
       "              (dropout): Dropout(p=0.1, inplace=False)\n",
       "            )\n",
       "          )\n",
       "          (intermediate): BertIntermediate(\n",
       "            (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
       "          )\n",
       "          (output): BertOutput(\n",
       "            (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
       "            (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
       "            (dropout): Dropout(p=0.1, inplace=False)\n",
       "          )\n",
       "        )\n",
       "        (4): BertLayer(\n",
       "          (attention): BertAttention(\n",
       "            (self): BertSelfAttention(\n",
       "              (query): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (key): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (value): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (dropout): Dropout(p=0.1, inplace=False)\n",
       "            )\n",
       "            (output): BertSelfOutput(\n",
       "              (dense): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
       "              (dropout): Dropout(p=0.1, inplace=False)\n",
       "            )\n",
       "          )\n",
       "          (intermediate): BertIntermediate(\n",
       "            (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
       "          )\n",
       "          (output): BertOutput(\n",
       "            (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
       "            (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
       "            (dropout): Dropout(p=0.1, inplace=False)\n",
       "          )\n",
       "        )\n",
       "        (5): BertLayer(\n",
       "          (attention): BertAttention(\n",
       "            (self): BertSelfAttention(\n",
       "              (query): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (key): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (value): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (dropout): Dropout(p=0.1, inplace=False)\n",
       "            )\n",
       "            (output): BertSelfOutput(\n",
       "              (dense): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
       "              (dropout): Dropout(p=0.1, inplace=False)\n",
       "            )\n",
       "          )\n",
       "          (intermediate): BertIntermediate(\n",
       "            (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
       "          )\n",
       "          (output): BertOutput(\n",
       "            (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
       "            (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
       "            (dropout): Dropout(p=0.1, inplace=False)\n",
       "          )\n",
       "        )\n",
       "        (6): BertLayer(\n",
       "          (attention): BertAttention(\n",
       "            (self): BertSelfAttention(\n",
       "              (query): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (key): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (value): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (dropout): Dropout(p=0.1, inplace=False)\n",
       "            )\n",
       "            (output): BertSelfOutput(\n",
       "              (dense): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
       "              (dropout): Dropout(p=0.1, inplace=False)\n",
       "            )\n",
       "          )\n",
       "          (intermediate): BertIntermediate(\n",
       "            (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
       "          )\n",
       "          (output): BertOutput(\n",
       "            (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
       "            (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
       "            (dropout): Dropout(p=0.1, inplace=False)\n",
       "          )\n",
       "        )\n",
       "        (7): BertLayer(\n",
       "          (attention): BertAttention(\n",
       "            (self): BertSelfAttention(\n",
       "              (query): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (key): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (value): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (dropout): Dropout(p=0.1, inplace=False)\n",
       "            )\n",
       "            (output): BertSelfOutput(\n",
       "              (dense): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
       "              (dropout): Dropout(p=0.1, inplace=False)\n",
       "            )\n",
       "          )\n",
       "          (intermediate): BertIntermediate(\n",
       "            (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
       "          )\n",
       "          (output): BertOutput(\n",
       "            (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
       "            (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
       "            (dropout): Dropout(p=0.1, inplace=False)\n",
       "          )\n",
       "        )\n",
       "        (8): BertLayer(\n",
       "          (attention): BertAttention(\n",
       "            (self): BertSelfAttention(\n",
       "              (query): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (key): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (value): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (dropout): Dropout(p=0.1, inplace=False)\n",
       "            )\n",
       "            (output): BertSelfOutput(\n",
       "              (dense): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
       "              (dropout): Dropout(p=0.1, inplace=False)\n",
       "            )\n",
       "          )\n",
       "          (intermediate): BertIntermediate(\n",
       "            (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
       "          )\n",
       "          (output): BertOutput(\n",
       "            (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
       "            (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
       "            (dropout): Dropout(p=0.1, inplace=False)\n",
       "          )\n",
       "        )\n",
       "        (9): BertLayer(\n",
       "          (attention): BertAttention(\n",
       "            (self): BertSelfAttention(\n",
       "              (query): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (key): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (value): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (dropout): Dropout(p=0.1, inplace=False)\n",
       "            )\n",
       "            (output): BertSelfOutput(\n",
       "              (dense): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
       "              (dropout): Dropout(p=0.1, inplace=False)\n",
       "            )\n",
       "          )\n",
       "          (intermediate): BertIntermediate(\n",
       "            (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
       "          )\n",
       "          (output): BertOutput(\n",
       "            (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
       "            (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
       "            (dropout): Dropout(p=0.1, inplace=False)\n",
       "          )\n",
       "        )\n",
       "        (10): BertLayer(\n",
       "          (attention): BertAttention(\n",
       "            (self): BertSelfAttention(\n",
       "              (query): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (key): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (value): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (dropout): Dropout(p=0.1, inplace=False)\n",
       "            )\n",
       "            (output): BertSelfOutput(\n",
       "              (dense): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
       "              (dropout): Dropout(p=0.1, inplace=False)\n",
       "            )\n",
       "          )\n",
       "          (intermediate): BertIntermediate(\n",
       "            (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
       "          )\n",
       "          (output): BertOutput(\n",
       "            (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
       "            (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
       "            (dropout): Dropout(p=0.1, inplace=False)\n",
       "          )\n",
       "        )\n",
       "        (11): BertLayer(\n",
       "          (attention): BertAttention(\n",
       "            (self): BertSelfAttention(\n",
       "              (query): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (key): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (value): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (dropout): Dropout(p=0.1, inplace=False)\n",
       "            )\n",
       "            (output): BertSelfOutput(\n",
       "              (dense): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
       "              (dropout): Dropout(p=0.1, inplace=False)\n",
       "            )\n",
       "          )\n",
       "          (intermediate): BertIntermediate(\n",
       "            (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
       "          )\n",
       "          (output): BertOutput(\n",
       "            (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
       "            (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
       "            (dropout): Dropout(p=0.1, inplace=False)\n",
       "          )\n",
       "        )\n",
       "      )\n",
       "    )\n",
       "    (pooler): BertPooler(\n",
       "      (dense): Linear(in_features=768, out_features=768, bias=True)\n",
       "      (activation): Tanh()\n",
       "    )\n",
       "  )\n",
       "  (classes): Linear(in_features=768, out_features=2, bias=True)\n",
       ")"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "classifier = Classifier(model)\n",
    "classifier.train()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Set up our optimizers\n",
    "Slower learning rate for Bert so we don't change it so much"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "bert_optimizer = torch.optim.Adam(classifier.encoder.parameters(), lr=0.000005)\n",
    "head_optimizer = torch.optim.Adam(classifier.classes.parameters(), lr=0.001)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Time to train the model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "STARTING EPOCH:  0\n",
      "Loss: 0.72\n",
      "Predictions: [0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0], Labels: [1 1 0 0 1 1 0 0 0 1 0 0 0 0 1 1 1 0 0 0], Accuracy: 0.6\n",
      "['Isn\\'t it weird that an actor is \"in\" a movie but \"on\" a TV show?', 'Love it when movies show you what it would be like to be someone else. Saw Last Train Home. Great movie about migrant workers in China.', 'Amazing how black socks change to blue when you take them out of the dresser drawer.', 'My kids saw a typewriter today and were blown away. \"What IS that thing?\" Like it was from an alien world.', \"I've largely cut sugar from my diet this week. Remember that Paul Giamatti movie Cold Souls? Me too.\", 'Recently watched the movie The Bay. Anyone want to go for a swim?', 'Coat hangers are very poorly behaved objects.', 'My inbox now has 0 messages. It feels good to get organized.', \"I wish my financial/retirement companies would stop sending me mail. With all these unnecessary words, I'm likely to miss important ones.\", 'Remember that kid\\'s movie Escape from Planet Earth? One character had the coolest coffee cup ever. It said, \"I love safety.\" I want that.', \"Man, how do people watch TV? You're sitting there all happy, and then suddenly someone tries to sell you something. Over, and over again.\", 'It would be nice to be able to cite videos in some permanent place.  Things disappear on YouTube.', 'Dreamed I played basketball with a giant strawberry for a ball. And there were South American soccer players camped out outside my window.', \"It's funny. In negotiations, the first person to specify a number loses, but the first person to establish a number as an anchor wins.\", \"Some movies don't seem to have enough material to fill up the trailer. Don't know what they do for the other 1 hour and 28 minutes.\", 'Just saw The Muppets. Interesting movie about a small fringe group fighting to keep America dependent on foreign oil.', 'Someday, we are going to look back and be embarrassed about the number of comic book movies made during this time period.', 'Sugar is the enemy. Sugar gives me life. #Productivity', 'Compared to the cold and hungry existence lived by our ancestors, our society is almost perfect. I tell myself that when I buy printer ink.', 'Wrestling with my boys. Good to have a little cave-man time before I go back to work tomorrow.']\n",
      "Loss: 0.74\n",
      "Predictions: [0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0], Labels: [1 0 0 1 1 0 0 0 0 0 1 0 0 1 1 1 0 1 0 1], Accuracy: 0.55\n",
      "['Finally watched Dr. Strangelove. Great movie, but the last scene with the character Dr. Strangelove was ... stupid.', 'Sprinkling unexpected words into your prose can keep it fresh but can also distract the reader. #Trade-offs #Writing', \"Hate when I can't find my mouse pointer. Even worse with two screens.\", 'Just got a wrong-number-call asking for a girl named Shirley. Always reminds me of that old movie Ruthless People.', 'Young people sometimes read the same books and watch the same movies multiple times. When we get older we stop doing that. Why?', \"Was just explaining @klout and how a person's influence is measured relative to the max of 100 held by Justin Bieber. This is our society.\", 'I learned a new word yesterday. aliteracy: people who can read but choose not to.', 'When younger, the soreness in my muscles after playing soccer was kind of cool. It meant I had worked hard. Now the soreness just hurts.', 'I\\'m used to \"twenty years ago\" meaning the 1970s, but that was twenty years ago.', 'Should the streets be widened to help emergency vehicles? Of course. Should my yard be shortened to widen the street? Get off my property.', \"Recently watched Anna Karenina. That movie would make no sense at all if I hadn't read the book.\", 'Network programming is somehow very satisfying. All of these happy little computers talking to each other.', \"I hate when I have to get up early. That's enough, but my body will often get me up even earlier than that for no good reason.\", 'One of my superpowers is the ability to abandon a book or movie halfway through.', \"A lot of my recurring dreams are like old movies that I don't quite remember. I know how they end, but I don't know how they'll get there.\", 'HT @TheOnion: 5-Year-Old Critics Agree: Movie \"Cars\" Only Gets Better After 40th Viewing', \"It's funny how deep questions and ignorant questions look alike. E.g., what is science?\", \"After having watched many Stars Wars movies with my kids lately, I've decided that R2-D2 is the unsung hero. The AI is strong in him.\", 'I love the smell of coffee in the morning. It smells like victory.', \"For picking movies, I've learned that not only must I enjoy it for 2 hours, but I must also enjoy having it in my head for days after.\"]\n",
      "Loss: 0.68\n",
      "Predictions: [0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0], Labels: [0 1 0 0 0 0 0 1 1 1 1 1 0 0 1 1 0 0 0 0], Accuracy: 0.6\n",
      "['I feel lucky to have a job where I learn every day. The world changes so fast--you have to run to keep up. #scary', 'Went to Costco for the first time not too long ago. That place is a disaster movie.', 'My daughter, age 3, kept asking me what time it was today. I guess she had somewhere to be.', 'I\\'ve been in the workforce for 15 years, and I still can\\'t get in the habit of saying \"good morning\" instead of \"hey\" or \"what\\'s up?\"', 'I love getting reminders from Microsoft Outlook telling me that I am 23 hours late for a meeting.', \"It's amazing how much joy in life comes from making and executing plans.\", 'Motto for the future of information: \"If it\\'s not relevant, get it out of my way.\"', 'I wonder why we raise our kids on movies with clear good guys and bad guys, when for the most part, the real world is just people', 'It would be funny to make a movie where characters argue using only maxims. Early bird gets the worm! Second mouse gets the cheese!', 'When they show scientists in movies, they never show them making PowerPoint slides.', \"Why can't we get good disaster movies?\", 'Told my wife I rented a movie about an automobile tire that comes to life and murders people. She just walked out of the room. #Marriage', 'I keep buying pyrite but I must be doing something wrong.', \"I hate when I write a note to myself but then later don't remember what the note refers to.\", \"It's weird how music makes everything more exciting like movies, tech videos, parties. It must tap into some deep, social part of the brain.\", \"I like a lot of kids movies, but I just can't get into these Ice Age films.\", 'Science moves forward by more accurate predictions. Math by new theorems. How does philosophy move forward?', 'Fame and fortune await the first person to develop a robot that powers itself by eating fire ants.', 'If it\\'s \"not necessary to dial a 1 when calling this number\" can\\'t the phone company just figure it out? Google has spoiled us all.', \"Surprisingly, the search feature on Google docs doesn't work very well. If only they had access to search experts.\"]\n",
      "Loss: 0.68\n",
      "Predictions: [0 1 0 0 0 0 0 0 0 0 0 0 1 0 0 0 0 0 0 0], Labels: [1 1 0 0 1 0 1 1 0 0 0 1 1 1 1 1 0 1 0 0], Accuracy: 0.55\n",
      "['Amazing how much of our time we spend in fantasy---movies, books, and TV. We live multiple simultaneous narratives.', \"Camping in the living room eating S'mores and watching a movie. It's surprising that they don't have more safety railings on the Death Star.\", \"Just got done working in the yard. We could solve the world's energy problems if we could somehow harness the power of cords to get tangled.\", \"My stomach is uncomfortably full, but I am still distracted by hunger. What's that story about the guy pushing the rock up the hill?\", 'Just saw Contagion. I thought it was pretty good. I like that it stayed with the big picture. Not as sappy as most disaster movies.', \"According to toddlers, a cat's tail was made for pulling. The perfect Gibsonian affordance.\", \"I've never been one to see symbolism in books or movies. I like to take fictional stories at face value, as if they were my own experience.\", 'Loved that movie Argo. It felt good to feel good about America, Hollywood, and Canada. #TwoThumbsUp', 'Office is out of coffee. Luckily, I have emergency backup. Kind of like the strategic petroleum reserve.', 'There is a tension in writing. You want to keep going to maintain momentum, but you also need to stop periodically to gain perspective.', 'Good comedy consists of statements that are both true and absurd.', 'Just saw Limitless. Great movie. But he said doubling your money every day was too slow. Clearly, even with NZT, he was still limited.', \"One bummer about getting older is that I've already seen every new movie that comes out.\", \"The movie I watched about a tire that comes to life and starts killing people wasn't as good as I had hoped it would be.\", 'The weird thing about the Lord of the Rings movies is that they all seem to just end. Like they ran out of film or something. #TheHobbit', 'Movies should bring back quicksand. We all need to be reminded of the dangers.', 'I coach kindergarten soccer. My standard April fools day joke is to email the parents about having two-a-day practices.', \"There must be a special place in hell for the people who decided you can't skip movie previews on DVD.\", 'Want evidence of intelligent machines? Captchas are now too hard for me.', \"I used to go one way to work. But lately, I've been asking Google Maps, and it sends me another way. I'm starting to see its wisdom.\"]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Loss: 0.73\n",
      "Predictions: [1 1 1 0 1 0 0 1 0 0 0 0 1 1 1 0 1 0 0 1], Labels: [0 1 0 1 0 1 0 0 1 1 0 1 1 0 0 0 0 0 0 1], Accuracy: 0.4\n",
      "[\"I successfully resurrected my wife's old laptop by installing Ubuntu. My work here is done.\", \"Finally saw Sleep Dealer (Traficante de Suenos). If you like dystopian movies about migrant works and drones, this one's for you. I loved it\", 'Finally saw Napoleon Dynamite. Love how you can always hear the birds chirping in the background. And the steaks.', 'Talking about the 6 Star Wars movies with my kids. Terms like \"first one\" and \"last one\" result in limitless confusion.', \"My son (7) wants to know why he can't have a credit card.\", \"I bet that primitive people used to just sit around and tell stories about monsters. Anyway, I'm off to the movies.\", 'It would be nice if we could abandon all of this inane talk about celebrities and go back to gossiping about people that we actually know.', \"Hate wearing a belt, but I need to dress fancy for work. I'm waiting for elastic belts to come back in fashion.\", 'I love it when movies about ghosts are \"based on a true story.\"', 'Just watched Tyrannosaur. That movie breaks your heart over and over again. Fewer dinosaurs than I expected.', 'I love how a good book becomes part of your mental life forever.', 'Just saw Black Swan. Man, that was stressful. I need to relax, maybe watch a movie or something.', \"With a big bucket of popcorn, it doesn't matter what the movie is. You're happy.\", \"Mute button didn't work on the box loudly squawking ads at the gas pump. Is the world not noisy enough? Those things are evil.\", 'Always surprised people keep those email ad signatures, such as \"Sent from my iPhone.\" Consider changing to \"Sent from my subconscious.\"', \"Companies should realize that surveys must be short. I'm not going to click on  25 ovals, I just want to say the hotel carpet was dirty.\", \"Last one in the office. I'm riding a scooter down an empty hall, and I'm suddenly reminded of the Big Wheel scene from The Shining.\", 'I was once at a party where someone spilled red wine, and they used white wine to clean it up. I wonder if the opposite would work.', 'I just wrote the word \"print\" and Google Docs flagged it as misspelled. It wanted me to write \"printf\". How cool is that?', 'I love reading the reviews for kids movies. \"... the archetypes do not resonate as one would expect given the ...\" Kids want burp jokes.']\n",
      "Loss: 0.70\n",
      "Predictions: [0], Labels: [1], Accuracy: 0.0\n",
      "['Evil Dead was last horror movie I saw. Never seen that kind of movie before. The brain is not meant to process that kind of sensory input.']\n",
      "STARTING EPOCH:  1\n",
      "Loss: 0.74\n",
      "Predictions: [1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1], Labels: [1 0 0 0 0 1 1 1 1 1 0 0 0 1 0 0 0 1 1 0], Accuracy: 0.45\n",
      "['Movies should bring back quicksand. We all need to be reminded of the dangers.', \"I hate when I write a note to myself but then later don't remember what the note refers to.\", \"It's funny how deep questions and ignorant questions look alike. E.g., what is science?\", 'Should the streets be widened to help emergency vehicles? Of course. Should my yard be shortened to widen the street? Get off my property.', \"I used to go one way to work. But lately, I've been asking Google Maps, and it sends me another way. I'm starting to see its wisdom.\", 'One of my superpowers is the ability to abandon a book or movie halfway through.', \"With a big bucket of popcorn, it doesn't matter what the movie is. You're happy.\", \"Some movies don't seem to have enough material to fill up the trailer. Don't know what they do for the other 1 hour and 28 minutes.\", \"I've never been one to see symbolism in books or movies. I like to take fictional stories at face value, as if they were my own experience.\", \"Camping in the living room eating S'mores and watching a movie. It's surprising that they don't have more safety railings on the Death Star.\", 'It would be nice to be able to cite videos in some permanent place.  Things disappear on YouTube.', 'I coach kindergarten soccer. My standard April fools day joke is to email the parents about having two-a-day practices.', 'Coat hangers are very poorly behaved objects.', \"Finally saw Sleep Dealer (Traficante de Suenos). If you like dystopian movies about migrant works and drones, this one's for you. I loved it\", \"It's amazing how much joy in life comes from making and executing plans.\", 'Motto for the future of information: \"If it\\'s not relevant, get it out of my way.\"', 'Wrestling with my boys. Good to have a little cave-man time before I go back to work tomorrow.', 'Loved that movie Argo. It felt good to feel good about America, Hollywood, and Canada. #TwoThumbsUp', 'HT @TheOnion: 5-Year-Old Critics Agree: Movie \"Cars\" Only Gets Better After 40th Viewing', 'If it\\'s \"not necessary to dial a 1 when calling this number\" can\\'t the phone company just figure it out? Google has spoiled us all.']\n",
      "Loss: 0.64\n",
      "Predictions: [1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1], Labels: [1 0 1 1 1 1 1 0 1 0 1 0 0 1 1 1 0 1 0 1], Accuracy: 0.65\n",
      "['Just saw Black Swan. Man, that was stressful. I need to relax, maybe watch a movie or something.', \"It's funny. In negotiations, the first person to specify a number loses, but the first person to establish a number as an anchor wins.\", 'Finally watched Dr. Strangelove. Great movie, but the last scene with the character Dr. Strangelove was ... stupid.', \"I've largely cut sugar from my diet this week. Remember that Paul Giamatti movie Cold Souls? Me too.\", 'When they show scientists in movies, they never show them making PowerPoint slides.', 'Recently watched the movie The Bay. Anyone want to go for a swim?', \"There must be a special place in hell for the people who decided you can't skip movie previews on DVD.\", \"Mute button didn't work on the box loudly squawking ads at the gas pump. Is the world not noisy enough? Those things are evil.\", 'I love reading the reviews for kids movies. \"... the archetypes do not resonate as one would expect given the ...\" Kids want burp jokes.', \"Hate when I can't find my mouse pointer. Even worse with two screens.\", 'Remember that kid\\'s movie Escape from Planet Earth? One character had the coolest coffee cup ever. It said, \"I love safety.\" I want that.', 'Always surprised people keep those email ad signatures, such as \"Sent from my iPhone.\" Consider changing to \"Sent from my subconscious.\"', 'Office is out of coffee. Luckily, I have emergency backup. Kind of like the strategic petroleum reserve.', 'Young people sometimes read the same books and watch the same movies multiple times. When we get older we stop doing that. Why?', \"The movie I watched about a tire that comes to life and starts killing people wasn't as good as I had hoped it would be.\", 'Isn\\'t it weird that an actor is \"in\" a movie but \"on\" a TV show?', 'Compared to the cold and hungry existence lived by our ancestors, our society is almost perfect. I tell myself that when I buy printer ink.', \"Recently watched Anna Karenina. That movie would make no sense at all if I hadn't read the book.\", 'Finally saw Napoleon Dynamite. Love how you can always hear the birds chirping in the background. And the steaks.', 'Someday, we are going to look back and be embarrassed about the number of comic book movies made during this time period.']\n",
      "Loss: 0.84\n",
      "Predictions: [1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1], Labels: [0 1 1 0 0 0 1 0 0 1 0 0 0 1 0 1 0 0 0 1], Accuracy: 0.35\n",
      "[\"My stomach is uncomfortably full, but I am still distracted by hunger. What's that story about the guy pushing the rock up the hill?\", 'Just saw The Muppets. Interesting movie about a small fringe group fighting to keep America dependent on foreign oil.', 'Talking about the 6 Star Wars movies with my kids. Terms like \"first one\" and \"last one\" result in limitless confusion.', \"Just got done working in the yard. We could solve the world's energy problems if we could somehow harness the power of cords to get tangled.\", 'I just wrote the word \"print\" and Google Docs flagged it as misspelled. It wanted me to write \"printf\". How cool is that?', 'I feel lucky to have a job where I learn every day. The world changes so fast--you have to run to keep up. #scary', 'Just got a wrong-number-call asking for a girl named Shirley. Always reminds me of that old movie Ruthless People.', 'My inbox now has 0 messages. It feels good to get organized.', 'My daughter, age 3, kept asking me what time it was today. I guess she had somewhere to be.', 'Evil Dead was last horror movie I saw. Never seen that kind of movie before. The brain is not meant to process that kind of sensory input.', 'I learned a new word yesterday. aliteracy: people who can read but choose not to.', 'I love getting reminders from Microsoft Outlook telling me that I am 23 hours late for a meeting.', 'Sugar is the enemy. Sugar gives me life. #Productivity', 'Told my wife I rented a movie about an automobile tire that comes to life and murders people. She just walked out of the room. #Marriage', 'Good comedy consists of statements that are both true and absurd.', 'It would be funny to make a movie where characters argue using only maxims. Early bird gets the worm! Second mouse gets the cheese!', 'Dreamed I played basketball with a giant strawberry for a ball. And there were South American soccer players camped out outside my window.', 'I was once at a party where someone spilled red wine, and they used white wine to clean it up. I wonder if the opposite would work.', 'My kids saw a typewriter today and were blown away. \"What IS that thing?\" Like it was from an alien world.', 'I wonder why we raise our kids on movies with clear good guys and bad guys, when for the most part, the real world is just people']\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Loss: 0.78\n",
      "Predictions: [1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1], Labels: [0 0 0 1 0 0 0 0 0 0 0 1 0 1 1 1 1 0 1 1], Accuracy: 0.4\n",
      "[\"I hate when I have to get up early. That's enough, but my body will often get me up even earlier than that for no good reason.\", 'Sprinkling unexpected words into your prose can keep it fresh but can also distract the reader. #Trade-offs #Writing', \"Last one in the office. I'm riding a scooter down an empty hall, and I'm suddenly reminded of the Big Wheel scene from The Shining.\", \"One bummer about getting older is that I've already seen every new movie that comes out.\", \"According to toddlers, a cat's tail was made for pulling. The perfect Gibsonian affordance.\", \"My son (7) wants to know why he can't have a credit card.\", 'Amazing how black socks change to blue when you take them out of the dresser drawer.', \"Surprisingly, the search feature on Google docs doesn't work very well. If only they had access to search experts.\", 'I\\'m used to \"twenty years ago\" meaning the 1970s, but that was twenty years ago.', 'Science moves forward by more accurate predictions. Math by new theorems. How does philosophy move forward?', 'Network programming is somehow very satisfying. All of these happy little computers talking to each other.', 'Amazing how much of our time we spend in fantasy---movies, books, and TV. We live multiple simultaneous narratives.', 'I\\'ve been in the workforce for 15 years, and I still can\\'t get in the habit of saying \"good morning\" instead of \"hey\" or \"what\\'s up?\"', 'The weird thing about the Lord of the Rings movies is that they all seem to just end. Like they ran out of film or something. #TheHobbit', 'Just saw Contagion. I thought it was pretty good. I like that it stayed with the big picture. Not as sappy as most disaster movies.', \"A lot of my recurring dreams are like old movies that I don't quite remember. I know how they end, but I don't know how they'll get there.\", 'Just watched Tyrannosaur. That movie breaks your heart over and over again. Fewer dinosaurs than I expected.', \"I successfully resurrected my wife's old laptop by installing Ubuntu. My work here is done.\", \"I like a lot of kids movies, but I just can't get into these Ice Age films.\", \"It's weird how music makes everything more exciting like movies, tech videos, parties. It must tap into some deep, social part of the brain.\"]\n",
      "Loss: 0.71\n",
      "Predictions: [0 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1], Labels: [0 1 1 0 0 1 0 1 0 1 1 0 0 1 0 1 0 0 0 0], Accuracy: 0.45\n",
      "['I love the smell of coffee in the morning. It smells like victory.', \"Why can't we get good disaster movies?\", 'Went to Costco for the first time not too long ago. That place is a disaster movie.', \"Man, how do people watch TV? You're sitting there all happy, and then suddenly someone tries to sell you something. Over, and over again.\", \"Hate wearing a belt, but I need to dress fancy for work. I'm waiting for elastic belts to come back in fashion.\", 'Love it when movies show you what it would be like to be someone else. Saw Last Train Home. Great movie about migrant workers in China.', 'It would be nice if we could abandon all of this inane talk about celebrities and go back to gossiping about people that we actually know.', 'Just saw Limitless. Great movie. But he said doubling your money every day was too slow. Clearly, even with NZT, he was still limited.', 'There is a tension in writing. You want to keep going to maintain momentum, but you also need to stop periodically to gain perspective.', 'I love it when movies about ghosts are \"based on a true story.\"', \"For picking movies, I've learned that not only must I enjoy it for 2 hours, but I must also enjoy having it in my head for days after.\", 'When younger, the soreness in my muscles after playing soccer was kind of cool. It meant I had worked hard. Now the soreness just hurts.', 'Want evidence of intelligent machines? Captchas are now too hard for me.', \"After having watched many Stars Wars movies with my kids lately, I've decided that R2-D2 is the unsung hero. The AI is strong in him.\", 'I love how a good book becomes part of your mental life forever.', \"I bet that primitive people used to just sit around and tell stories about monsters. Anyway, I'm off to the movies.\", 'I keep buying pyrite but I must be doing something wrong.', \"I wish my financial/retirement companies would stop sending me mail. With all these unnecessary words, I'm likely to miss important ones.\", \"Companies should realize that surveys must be short. I'm not going to click on  25 ovals, I just want to say the hotel carpet was dirty.\", 'Fame and fortune await the first person to develop a robot that powers itself by eating fire ants.']\n",
      "Loss: 0.90\n",
      "Predictions: [1], Labels: [0], Accuracy: 0.0\n",
      "[\"Was just explaining @klout and how a person's influence is measured relative to the max of 100 held by Justin Bieber. This is our society.\"]\n",
      "STARTING EPOCH:  2\n",
      "Loss: 0.69\n",
      "Predictions: [1 1 0 0 0 0 1 0 0 0 0 0 0 0 0 1 0 0 0 0], Labels: [0 1 0 0 0 1 1 0 1 0 1 1 0 1 0 1 0 0 1 0], Accuracy: 0.65\n",
      "['I was once at a party where someone spilled red wine, and they used white wine to clean it up. I wonder if the opposite would work.', 'Someday, we are going to look back and be embarrassed about the number of comic book movies made during this time period.', 'Network programming is somehow very satisfying. All of these happy little computers talking to each other.', \"My son (7) wants to know why he can't have a credit card.\", \"Hate when I can't find my mouse pointer. Even worse with two screens.\", 'Finally watched Dr. Strangelove. Great movie, but the last scene with the character Dr. Strangelove was ... stupid.', 'Just saw Contagion. I thought it was pretty good. I like that it stayed with the big picture. Not as sappy as most disaster movies.', 'There is a tension in writing. You want to keep going to maintain momentum, but you also need to stop periodically to gain perspective.', \"I bet that primitive people used to just sit around and tell stories about monsters. Anyway, I'm off to the movies.\", \"I used to go one way to work. But lately, I've been asking Google Maps, and it sends me another way. I'm starting to see its wisdom.\", \"The movie I watched about a tire that comes to life and starts killing people wasn't as good as I had hoped it would be.\", \"Finally saw Sleep Dealer (Traficante de Suenos). If you like dystopian movies about migrant works and drones, this one's for you. I loved it\", 'Coat hangers are very poorly behaved objects.', \"Some movies don't seem to have enough material to fill up the trailer. Don't know what they do for the other 1 hour and 28 minutes.\", 'Sprinkling unexpected words into your prose can keep it fresh but can also distract the reader. #Trade-offs #Writing', 'Amazing how much of our time we spend in fantasy---movies, books, and TV. We live multiple simultaneous narratives.', \"It's funny. In negotiations, the first person to specify a number loses, but the first person to establish a number as an anchor wins.\", 'I love how a good book becomes part of your mental life forever.', 'Movies should bring back quicksand. We all need to be reminded of the dangers.', 'Fame and fortune await the first person to develop a robot that powers itself by eating fire ants.']\n",
      "Loss: 0.61\n",
      "Predictions: [0 0 0 0 0 0 0 0 1 0 0 0 0 0 0 0 0 0 0 0], Labels: [1 0 0 0 0 0 1 0 1 0 0 0 0 1 1 0 0 1 1 0], Accuracy: 0.7\n",
      "['Evil Dead was last horror movie I saw. Never seen that kind of movie before. The brain is not meant to process that kind of sensory input.', 'My kids saw a typewriter today and were blown away. \"What IS that thing?\" Like it was from an alien world.', 'I coach kindergarten soccer. My standard April fools day joke is to email the parents about having two-a-day practices.', 'Motto for the future of information: \"If it\\'s not relevant, get it out of my way.\"', 'Dreamed I played basketball with a giant strawberry for a ball. And there were South American soccer players camped out outside my window.', 'Finally saw Napoleon Dynamite. Love how you can always hear the birds chirping in the background. And the steaks.', 'Just saw Black Swan. Man, that was stressful. I need to relax, maybe watch a movie or something.', 'I love the smell of coffee in the morning. It smells like victory.', \"Why can't we get good disaster movies?\", 'Always surprised people keep those email ad signatures, such as \"Sent from my iPhone.\" Consider changing to \"Sent from my subconscious.\"', 'My daughter, age 3, kept asking me what time it was today. I guess she had somewhere to be.', \"Last one in the office. I'm riding a scooter down an empty hall, and I'm suddenly reminded of the Big Wheel scene from The Shining.\", 'I keep buying pyrite but I must be doing something wrong.', \"It's weird how music makes everything more exciting like movies, tech videos, parties. It must tap into some deep, social part of the brain.\", 'Isn\\'t it weird that an actor is \"in\" a movie but \"on\" a TV show?', 'Should the streets be widened to help emergency vehicles? Of course. Should my yard be shortened to widen the street? Get off my property.', 'I learned a new word yesterday. aliteracy: people who can read but choose not to.', \"With a big bucket of popcorn, it doesn't matter what the movie is. You're happy.\", 'Remember that kid\\'s movie Escape from Planet Earth? One character had the coolest coffee cup ever. It said, \"I love safety.\" I want that.', \"Surprisingly, the search feature on Google docs doesn't work very well. If only they had access to search experts.\"]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Loss: 0.68\n",
      "Predictions: [0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 1 0], Labels: [0 1 1 0 1 0 1 0 1 1 0 1 0 0 0 0 0 1 1 0], Accuracy: 0.6\n",
      "[\"I hate when I write a note to myself but then later don't remember what the note refers to.\", \"After having watched many Stars Wars movies with my kids lately, I've decided that R2-D2 is the unsung hero. The AI is strong in him.\", \"I like a lot of kids movies, but I just can't get into these Ice Age films.\", \"Companies should realize that surveys must be short. I'm not going to click on  25 ovals, I just want to say the hotel carpet was dirty.\", \"I've largely cut sugar from my diet this week. Remember that Paul Giamatti movie Cold Souls? Me too.\", 'I\\'m used to \"twenty years ago\" meaning the 1970s, but that was twenty years ago.', 'Told my wife I rented a movie about an automobile tire that comes to life and murders people. She just walked out of the room. #Marriage', \"Was just explaining @klout and how a person's influence is measured relative to the max of 100 held by Justin Bieber. This is our society.\", 'I love reading the reviews for kids movies. \"... the archetypes do not resonate as one would expect given the ...\" Kids want burp jokes.', \"I've never been one to see symbolism in books or movies. I like to take fictional stories at face value, as if they were my own experience.\", 'I feel lucky to have a job where I learn every day. The world changes so fast--you have to run to keep up. #scary', 'When they show scientists in movies, they never show them making PowerPoint slides.', 'When younger, the soreness in my muscles after playing soccer was kind of cool. It meant I had worked hard. Now the soreness just hurts.', \"I hate when I have to get up early. That's enough, but my body will often get me up even earlier than that for no good reason.\", 'I\\'ve been in the workforce for 15 years, and I still can\\'t get in the habit of saying \"good morning\" instead of \"hey\" or \"what\\'s up?\"', \"It's amazing how much joy in life comes from making and executing plans.\", 'I love getting reminders from Microsoft Outlook telling me that I am 23 hours late for a meeting.', \"A lot of my recurring dreams are like old movies that I don't quite remember. I know how they end, but I don't know how they'll get there.\", 'Recently watched the movie The Bay. Anyone want to go for a swim?', 'Wrestling with my boys. Good to have a little cave-man time before I go back to work tomorrow.']\n",
      "Loss: 0.77\n",
      "Predictions: [0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0], Labels: [1 1 0 0 1 1 1 0 1 0 0 0 0 1 1 1 0 1 0 1], Accuracy: 0.45\n",
      "['Loved that movie Argo. It felt good to feel good about America, Hollywood, and Canada. #TwoThumbsUp', 'Talking about the 6 Star Wars movies with my kids. Terms like \"first one\" and \"last one\" result in limitless confusion.', \"Mute button didn't work on the box loudly squawking ads at the gas pump. Is the world not noisy enough? Those things are evil.\", \"Man, how do people watch TV? You're sitting there all happy, and then suddenly someone tries to sell you something. Over, and over again.\", 'Just saw Limitless. Great movie. But he said doubling your money every day was too slow. Clearly, even with NZT, he was still limited.', \"There must be a special place in hell for the people who decided you can't skip movie previews on DVD.\", 'HT @TheOnion: 5-Year-Old Critics Agree: Movie \"Cars\" Only Gets Better After 40th Viewing', 'Sugar is the enemy. Sugar gives me life. #Productivity', \"For picking movies, I've learned that not only must I enjoy it for 2 hours, but I must also enjoy having it in my head for days after.\", 'If it\\'s \"not necessary to dial a 1 when calling this number\" can\\'t the phone company just figure it out? Google has spoiled us all.', \"I wish my financial/retirement companies would stop sending me mail. With all these unnecessary words, I'm likely to miss important ones.\", 'Want evidence of intelligent machines? Captchas are now too hard for me.', 'It would be nice if we could abandon all of this inane talk about celebrities and go back to gossiping about people that we actually know.', \"One bummer about getting older is that I've already seen every new movie that comes out.\", 'One of my superpowers is the ability to abandon a book or movie halfway through.', \"Recently watched Anna Karenina. That movie would make no sense at all if I hadn't read the book.\", \"Hate wearing a belt, but I need to dress fancy for work. I'm waiting for elastic belts to come back in fashion.\", 'The weird thing about the Lord of the Rings movies is that they all seem to just end. Like they ran out of film or something. #TheHobbit', \"According to toddlers, a cat's tail was made for pulling. The perfect Gibsonian affordance.\", 'Just saw The Muppets. Interesting movie about a small fringe group fighting to keep America dependent on foreign oil.']\n",
      "Loss: 0.74\n",
      "Predictions: [0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0], Labels: [1 1 0 0 0 0 0 1 0 1 1 1 0 1 1 0 0 0 0 1], Accuracy: 0.55\n",
      "['Went to Costco for the first time not too long ago. That place is a disaster movie.', 'I love it when movies about ghosts are \"based on a true story.\"', \"Just got done working in the yard. We could solve the world's energy problems if we could somehow harness the power of cords to get tangled.\", \"My stomach is uncomfortably full, but I am still distracted by hunger. What's that story about the guy pushing the rock up the hill?\", \"It's funny how deep questions and ignorant questions look alike. E.g., what is science?\", 'I just wrote the word \"print\" and Google Docs flagged it as misspelled. It wanted me to write \"printf\". How cool is that?', 'My inbox now has 0 messages. It feels good to get organized.', 'Just got a wrong-number-call asking for a girl named Shirley. Always reminds me of that old movie Ruthless People.', 'Office is out of coffee. Luckily, I have emergency backup. Kind of like the strategic petroleum reserve.', 'I wonder why we raise our kids on movies with clear good guys and bad guys, when for the most part, the real world is just people', 'Young people sometimes read the same books and watch the same movies multiple times. When we get older we stop doing that. Why?', 'It would be funny to make a movie where characters argue using only maxims. Early bird gets the worm! Second mouse gets the cheese!', \"I successfully resurrected my wife's old laptop by installing Ubuntu. My work here is done.\", \"Camping in the living room eating S'mores and watching a movie. It's surprising that they don't have more safety railings on the Death Star.\", 'Love it when movies show you what it would be like to be someone else. Saw Last Train Home. Great movie about migrant workers in China.', 'It would be nice to be able to cite videos in some permanent place.  Things disappear on YouTube.', 'Compared to the cold and hungry existence lived by our ancestors, our society is almost perfect. I tell myself that when I buy printer ink.', 'Amazing how black socks change to blue when you take them out of the dresser drawer.', 'Good comedy consists of statements that are both true and absurd.', 'Just watched Tyrannosaur. That movie breaks your heart over and over again. Fewer dinosaurs than I expected.']\n",
      "Loss: 0.18\n",
      "Predictions: [0], Labels: [0], Accuracy: 1.0\n",
      "['Science moves forward by more accurate predictions. Math by new theorems. How does philosophy move forward?']\n",
      "STARTING EPOCH:  3\n",
      "Loss: 0.78\n",
      "Predictions: [0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 1 0 0 0], Labels: [0 1 0 0 1 0 0 1 0 1 0 1 1 1 0 1 0 0 1 0], Accuracy: 0.5\n",
      "[\"I hate when I write a note to myself but then later don't remember what the note refers to.\", \"Camping in the living room eating S'mores and watching a movie. It's surprising that they don't have more safety railings on the Death Star.\", \"Was just explaining @klout and how a person's influence is measured relative to the max of 100 held by Justin Bieber. This is our society.\", 'Want evidence of intelligent machines? Captchas are now too hard for me.', 'Just saw Contagion. I thought it was pretty good. I like that it stayed with the big picture. Not as sappy as most disaster movies.', \"Surprisingly, the search feature on Google docs doesn't work very well. If only they had access to search experts.\", 'Coat hangers are very poorly behaved objects.', \"After having watched many Stars Wars movies with my kids lately, I've decided that R2-D2 is the unsung hero. The AI is strong in him.\", 'My kids saw a typewriter today and were blown away. \"What IS that thing?\" Like it was from an alien world.', \"A lot of my recurring dreams are like old movies that I don't quite remember. I know how they end, but I don't know how they'll get there.\", \"Last one in the office. I'm riding a scooter down an empty hall, and I'm suddenly reminded of the Big Wheel scene from The Shining.\", \"I bet that primitive people used to just sit around and tell stories about monsters. Anyway, I'm off to the movies.\", \"Why can't we get good disaster movies?\", \"It's weird how music makes everything more exciting like movies, tech videos, parties. It must tap into some deep, social part of the brain.\", 'Science moves forward by more accurate predictions. Math by new theorems. How does philosophy move forward?', 'When they show scientists in movies, they never show them making PowerPoint slides.', 'I was once at a party where someone spilled red wine, and they used white wine to clean it up. I wonder if the opposite would work.', 'I learned a new word yesterday. aliteracy: people who can read but choose not to.', 'Someday, we are going to look back and be embarrassed about the number of comic book movies made during this time period.', \"Mute button didn't work on the box loudly squawking ads at the gas pump. Is the world not noisy enough? Those things are evil.\"]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Loss: 0.76\n",
      "Predictions: [0 0 0 0 1 0 0 0 0 0 0 0 0 0 0 0 0 0 0 1], Labels: [0 0 1 0 1 0 1 1 0 0 0 0 0 1 1 0 1 1 1 0], Accuracy: 0.55\n",
      "['I love the smell of coffee in the morning. It smells like victory.', 'My inbox now has 0 messages. It feels good to get organized.', \"The movie I watched about a tire that comes to life and starts killing people wasn't as good as I had hoped it would be.\", 'I love how a good book becomes part of your mental life forever.', 'Isn\\'t it weird that an actor is \"in\" a movie but \"on\" a TV show?', \"My stomach is uncomfortably full, but I am still distracted by hunger. What's that story about the guy pushing the rock up the hill?\", 'The weird thing about the Lord of the Rings movies is that they all seem to just end. Like they ran out of film or something. #TheHobbit', \"There must be a special place in hell for the people who decided you can't skip movie previews on DVD.\", \"I hate when I have to get up early. That's enough, but my body will often get me up even earlier than that for no good reason.\", \"Just got done working in the yard. We could solve the world's energy problems if we could somehow harness the power of cords to get tangled.\", \"Companies should realize that surveys must be short. I'm not going to click on  25 ovals, I just want to say the hotel carpet was dirty.\", 'I keep buying pyrite but I must be doing something wrong.', \"I successfully resurrected my wife's old laptop by installing Ubuntu. My work here is done.\", 'It would be funny to make a movie where characters argue using only maxims. Early bird gets the worm! Second mouse gets the cheese!', 'Went to Costco for the first time not too long ago. That place is a disaster movie.', 'I\\'m used to \"twenty years ago\" meaning the 1970s, but that was twenty years ago.', 'Told my wife I rented a movie about an automobile tire that comes to life and murders people. She just walked out of the room. #Marriage', 'Just watched Tyrannosaur. That movie breaks your heart over and over again. Fewer dinosaurs than I expected.', 'Just got a wrong-number-call asking for a girl named Shirley. Always reminds me of that old movie Ruthless People.', 'Fame and fortune await the first person to develop a robot that powers itself by eating fire ants.']\n",
      "Loss: 0.57\n",
      "Predictions: [0 0 1 0 0 0 0 1 0 0 0 0 0 0 0 0 1 0 0 0], Labels: [0 0 1 1 1 0 0 1 0 1 0 0 0 0 1 1 1 0 0 0], Accuracy: 0.75\n",
      "['Compared to the cold and hungry existence lived by our ancestors, our society is almost perfect. I tell myself that when I buy printer ink.', \"I used to go one way to work. But lately, I've been asking Google Maps, and it sends me another way. I'm starting to see its wisdom.\", 'Just saw Black Swan. Man, that was stressful. I need to relax, maybe watch a movie or something.', \"One bummer about getting older is that I've already seen every new movie that comes out.\", 'One of my superpowers is the ability to abandon a book or movie halfway through.', 'It would be nice if we could abandon all of this inane talk about celebrities and go back to gossiping about people that we actually know.', 'I\\'ve been in the workforce for 15 years, and I still can\\'t get in the habit of saying \"good morning\" instead of \"hey\" or \"what\\'s up?\"', \"I like a lot of kids movies, but I just can't get into these Ice Age films.\", 'Sugar is the enemy. Sugar gives me life. #Productivity', 'Talking about the 6 Star Wars movies with my kids. Terms like \"first one\" and \"last one\" result in limitless confusion.', 'Office is out of coffee. Luckily, I have emergency backup. Kind of like the strategic petroleum reserve.', 'It would be nice to be able to cite videos in some permanent place.  Things disappear on YouTube.', 'Always surprised people keep those email ad signatures, such as \"Sent from my iPhone.\" Consider changing to \"Sent from my subconscious.\"', 'My daughter, age 3, kept asking me what time it was today. I guess she had somewhere to be.', 'I love it when movies about ghosts are \"based on a true story.\"', 'Just saw Limitless. Great movie. But he said doubling your money every day was too slow. Clearly, even with NZT, he was still limited.', 'Recently watched the movie The Bay. Anyone want to go for a swim?', 'I love getting reminders from Microsoft Outlook telling me that I am 23 hours late for a meeting.', 'I coach kindergarten soccer. My standard April fools day joke is to email the parents about having two-a-day practices.', 'Sprinkling unexpected words into your prose can keep it fresh but can also distract the reader. #Trade-offs #Writing']\n",
      "Loss: 0.64\n",
      "Predictions: [0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0], Labels: [0 0 0 0 1 0 1 0 1 0 0 0 1 0 1 1 1 0 1 0], Accuracy: 0.6\n",
      "['I feel lucky to have a job where I learn every day. The world changes so fast--you have to run to keep up. #scary', \"Man, how do people watch TV? You're sitting there all happy, and then suddenly someone tries to sell you something. Over, and over again.\", \"I wish my financial/retirement companies would stop sending me mail. With all these unnecessary words, I'm likely to miss important ones.\", 'Amazing how black socks change to blue when you take them out of the dresser drawer.', 'I wonder why we raise our kids on movies with clear good guys and bad guys, when for the most part, the real world is just people', 'Wrestling with my boys. Good to have a little cave-man time before I go back to work tomorrow.', \"I've never been one to see symbolism in books or movies. I like to take fictional stories at face value, as if they were my own experience.\", \"It's funny how deep questions and ignorant questions look alike. E.g., what is science?\", \"Finally saw Sleep Dealer (Traficante de Suenos). If you like dystopian movies about migrant works and drones, this one's for you. I loved it\", 'Should the streets be widened to help emergency vehicles? Of course. Should my yard be shortened to widen the street? Get off my property.', 'Network programming is somehow very satisfying. All of these happy little computers talking to each other.', \"It's funny. In negotiations, the first person to specify a number loses, but the first person to establish a number as an anchor wins.\", 'I love reading the reviews for kids movies. \"... the archetypes do not resonate as one would expect given the ...\" Kids want burp jokes.', 'When younger, the soreness in my muscles after playing soccer was kind of cool. It meant I had worked hard. Now the soreness just hurts.', 'Remember that kid\\'s movie Escape from Planet Earth? One character had the coolest coffee cup ever. It said, \"I love safety.\" I want that.', 'Movies should bring back quicksand. We all need to be reminded of the dangers.', 'Evil Dead was last horror movie I saw. Never seen that kind of movie before. The brain is not meant to process that kind of sensory input.', 'Motto for the future of information: \"If it\\'s not relevant, get it out of my way.\"', 'Finally watched Dr. Strangelove. Great movie, but the last scene with the character Dr. Strangelove was ... stupid.', \"Hate wearing a belt, but I need to dress fancy for work. I'm waiting for elastic belts to come back in fashion.\"]\n",
      "Loss: 0.79\n",
      "Predictions: [0 1 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 1 0 0], Labels: [1 0 1 0 0 0 1 0 1 1 1 0 0 1 0 1 1 0 0 1], Accuracy: 0.4\n",
      "[\"I've largely cut sugar from my diet this week. Remember that Paul Giamatti movie Cold Souls? Me too.\", \"My son (7) wants to know why he can't have a credit card.\", \"For picking movies, I've learned that not only must I enjoy it for 2 hours, but I must also enjoy having it in my head for days after.\", \"According to toddlers, a cat's tail was made for pulling. The perfect Gibsonian affordance.\", 'I just wrote the word \"print\" and Google Docs flagged it as misspelled. It wanted me to write \"printf\". How cool is that?', 'If it\\'s \"not necessary to dial a 1 when calling this number\" can\\'t the phone company just figure it out? Google has spoiled us all.', 'Young people sometimes read the same books and watch the same movies multiple times. When we get older we stop doing that. Why?', \"Hate when I can't find my mouse pointer. Even worse with two screens.\", 'Loved that movie Argo. It felt good to feel good about America, Hollywood, and Canada. #TwoThumbsUp', \"Some movies don't seem to have enough material to fill up the trailer. Don't know what they do for the other 1 hour and 28 minutes.\", \"With a big bucket of popcorn, it doesn't matter what the movie is. You're happy.\", 'Finally saw Napoleon Dynamite. Love how you can always hear the birds chirping in the background. And the steaks.', 'There is a tension in writing. You want to keep going to maintain momentum, but you also need to stop periodically to gain perspective.', \"Recently watched Anna Karenina. That movie would make no sense at all if I hadn't read the book.\", 'Dreamed I played basketball with a giant strawberry for a ball. And there were South American soccer players camped out outside my window.', 'Amazing how much of our time we spend in fantasy---movies, books, and TV. We live multiple simultaneous narratives.', 'Just saw The Muppets. Interesting movie about a small fringe group fighting to keep America dependent on foreign oil.', \"It's amazing how much joy in life comes from making and executing plans.\", 'Good comedy consists of statements that are both true and absurd.', 'HT @TheOnion: 5-Year-Old Critics Agree: Movie \"Cars\" Only Gets Better After 40th Viewing']\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Loss: 0.79\n",
      "Predictions: [0], Labels: [1], Accuracy: 0.0\n",
      "['Love it when movies show you what it would be like to be someone else. Saw Last Train Home. Great movie about migrant workers in China.']\n",
      "STARTING EPOCH:  4\n",
      "Loss: 0.58\n",
      "Predictions: [1 0 0 1 0 0 0 0 0 0 1 0 0 1 0 0 0 1 0 0], Labels: [1 0 0 0 1 1 1 1 0 0 1 1 1 1 0 0 0 1 0 0], Accuracy: 0.65\n",
      "['Recently watched the movie The Bay. Anyone want to go for a swim?', 'When younger, the soreness in my muscles after playing soccer was kind of cool. It meant I had worked hard. Now the soreness just hurts.', 'Fame and fortune await the first person to develop a robot that powers itself by eating fire ants.', 'Office is out of coffee. Luckily, I have emergency backup. Kind of like the strategic petroleum reserve.', 'Someday, we are going to look back and be embarrassed about the number of comic book movies made during this time period.', \"For picking movies, I've learned that not only must I enjoy it for 2 hours, but I must also enjoy having it in my head for days after.\", 'Amazing how much of our time we spend in fantasy---movies, books, and TV. We live multiple simultaneous narratives.', \"With a big bucket of popcorn, it doesn't matter what the movie is. You're happy.\", 'I just wrote the word \"print\" and Google Docs flagged it as misspelled. It wanted me to write \"printf\". How cool is that?', 'I\\'m used to \"twenty years ago\" meaning the 1970s, but that was twenty years ago.', 'Love it when movies show you what it would be like to be someone else. Saw Last Train Home. Great movie about migrant workers in China.', 'It would be funny to make a movie where characters argue using only maxims. Early bird gets the worm! Second mouse gets the cheese!', \"I like a lot of kids movies, but I just can't get into these Ice Age films.\", 'Told my wife I rented a movie about an automobile tire that comes to life and murders people. She just walked out of the room. #Marriage', 'It would be nice if we could abandon all of this inane talk about celebrities and go back to gossiping about people that we actually know.', \"It's funny. In negotiations, the first person to specify a number loses, but the first person to establish a number as an anchor wins.\", 'There is a tension in writing. You want to keep going to maintain momentum, but you also need to stop periodically to gain perspective.', 'Finally watched Dr. Strangelove. Great movie, but the last scene with the character Dr. Strangelove was ... stupid.', \"Surprisingly, the search feature on Google docs doesn't work very well. If only they had access to search experts.\", \"Man, how do people watch TV? You're sitting there all happy, and then suddenly someone tries to sell you something. Over, and over again.\"]\n",
      "Loss: 0.60\n",
      "Predictions: [0 0 1 0 0 1 0 0 0 1 1 1 0 0 0 0 1 0 1 0], Labels: [1 0 1 0 0 1 0 1 0 1 1 1 0 0 0 1 1 1 0 0], Accuracy: 0.75\n",
      "[\"I've largely cut sugar from my diet this week. Remember that Paul Giamatti movie Cold Souls? Me too.\", \"According to toddlers, a cat's tail was made for pulling. The perfect Gibsonian affordance.\", \"Camping in the living room eating S'mores and watching a movie. It's surprising that they don't have more safety railings on the Death Star.\", 'If it\\'s \"not necessary to dial a 1 when calling this number\" can\\'t the phone company just figure it out? Google has spoiled us all.', \"Mute button didn't work on the box loudly squawking ads at the gas pump. Is the world not noisy enough? Those things are evil.\", \"Finally saw Sleep Dealer (Traficante de Suenos). If you like dystopian movies about migrant works and drones, this one's for you. I loved it\", \"Companies should realize that surveys must be short. I'm not going to click on  25 ovals, I just want to say the hotel carpet was dirty.\", 'One of my superpowers is the ability to abandon a book or movie halfway through.', 'Always surprised people keep those email ad signatures, such as \"Sent from my iPhone.\" Consider changing to \"Sent from my subconscious.\"', \"There must be a special place in hell for the people who decided you can't skip movie previews on DVD.\", 'Evil Dead was last horror movie I saw. Never seen that kind of movie before. The brain is not meant to process that kind of sensory input.', 'Just got a wrong-number-call asking for a girl named Shirley. Always reminds me of that old movie Ruthless People.', 'I\\'ve been in the workforce for 15 years, and I still can\\'t get in the habit of saying \"good morning\" instead of \"hey\" or \"what\\'s up?\"', 'I keep buying pyrite but I must be doing something wrong.', \"My son (7) wants to know why he can't have a credit card.\", \"I bet that primitive people used to just sit around and tell stories about monsters. Anyway, I'm off to the movies.\", 'Talking about the 6 Star Wars movies with my kids. Terms like \"first one\" and \"last one\" result in limitless confusion.', 'Movies should bring back quicksand. We all need to be reminded of the dangers.', 'Coat hangers are very poorly behaved objects.', 'Good comedy consists of statements that are both true and absurd.']\n",
      "Loss: 0.51\n",
      "Predictions: [0 0 1 0 1 1 0 1 0 0 1 0 0 0 0 1 1 0 0 0], Labels: [0 0 1 0 1 1 0 1 0 0 1 1 0 0 0 1 0 0 0 0], Accuracy: 0.9\n",
      "[\"It's amazing how much joy in life comes from making and executing plans.\", \"I wish my financial/retirement companies would stop sending me mail. With all these unnecessary words, I'm likely to miss important ones.\", \"One bummer about getting older is that I've already seen every new movie that comes out.\", 'I feel lucky to have a job where I learn every day. The world changes so fast--you have to run to keep up. #scary', 'When they show scientists in movies, they never show them making PowerPoint slides.', 'Remember that kid\\'s movie Escape from Planet Earth? One character had the coolest coffee cup ever. It said, \"I love safety.\" I want that.', 'My kids saw a typewriter today and were blown away. \"What IS that thing?\" Like it was from an alien world.', 'Isn\\'t it weird that an actor is \"in\" a movie but \"on\" a TV show?', 'I love how a good book becomes part of your mental life forever.', \"I hate when I write a note to myself but then later don't remember what the note refers to.\", 'The weird thing about the Lord of the Rings movies is that they all seem to just end. Like they ran out of film or something. #TheHobbit', 'I love reading the reviews for kids movies. \"... the archetypes do not resonate as one would expect given the ...\" Kids want burp jokes.', \"Hate wearing a belt, but I need to dress fancy for work. I'm waiting for elastic belts to come back in fashion.\", 'Wrestling with my boys. Good to have a little cave-man time before I go back to work tomorrow.', 'My inbox now has 0 messages. It feels good to get organized.', \"Recently watched Anna Karenina. That movie would make no sense at all if I hadn't read the book.\", 'Want evidence of intelligent machines? Captchas are now too hard for me.', \"I hate when I have to get up early. That's enough, but my body will often get me up even earlier than that for no good reason.\", 'I was once at a party where someone spilled red wine, and they used white wine to clean it up. I wonder if the opposite would work.', \"I used to go one way to work. But lately, I've been asking Google Maps, and it sends me another way. I'm starting to see its wisdom.\"]\n",
      "Loss: 0.56\n",
      "Predictions: [0 0 1 1 1 0 0 1 1 1 1 0 1 1 0 1 1 0 0 1], Labels: [1 0 0 0 1 0 0 1 1 0 1 0 1 1 0 0 0 0 0 1], Accuracy: 0.7\n",
      "['Loved that movie Argo. It felt good to feel good about America, Hollywood, and Canada. #TwoThumbsUp', \"Hate when I can't find my mouse pointer. Even worse with two screens.\", 'Network programming is somehow very satisfying. All of these happy little computers talking to each other.', 'Finally saw Napoleon Dynamite. Love how you can always hear the birds chirping in the background. And the steaks.', 'Just saw The Muppets. Interesting movie about a small fringe group fighting to keep America dependent on foreign oil.', 'Motto for the future of information: \"If it\\'s not relevant, get it out of my way.\"', 'Science moves forward by more accurate predictions. Math by new theorems. How does philosophy move forward?', 'Just saw Black Swan. Man, that was stressful. I need to relax, maybe watch a movie or something.', 'Went to Costco for the first time not too long ago. That place is a disaster movie.', \"It's funny how deep questions and ignorant questions look alike. E.g., what is science?\", \"Some movies don't seem to have enough material to fill up the trailer. Don't know what they do for the other 1 hour and 28 minutes.\", 'Sugar is the enemy. Sugar gives me life. #Productivity', 'Just saw Contagion. I thought it was pretty good. I like that it stayed with the big picture. Not as sappy as most disaster movies.', \"A lot of my recurring dreams are like old movies that I don't quite remember. I know how they end, but I don't know how they'll get there.\", 'I love getting reminders from Microsoft Outlook telling me that I am 23 hours late for a meeting.', \"My stomach is uncomfortably full, but I am still distracted by hunger. What's that story about the guy pushing the rock up the hill?\", \"Last one in the office. I'm riding a scooter down an empty hall, and I'm suddenly reminded of the Big Wheel scene from The Shining.\", 'Should the streets be widened to help emergency vehicles? Of course. Should my yard be shortened to widen the street? Get off my property.', \"I successfully resurrected my wife's old laptop by installing Ubuntu. My work here is done.\", 'HT @TheOnion: 5-Year-Old Critics Agree: Movie \"Cars\" Only Gets Better After 40th Viewing']\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Loss: 0.63\n",
      "Predictions: [0 1 1 1 1 1 0 0 1 0 1 0 1 1 1 0 1 0 0 1], Labels: [0 0 1 1 0 0 0 1 0 0 1 0 1 0 1 1 1 0 0 1], Accuracy: 0.65\n",
      "['Amazing how black socks change to blue when you take them out of the dresser drawer.', 'I coach kindergarten soccer. My standard April fools day joke is to email the parents about having two-a-day practices.', \"After having watched many Stars Wars movies with my kids lately, I've decided that R2-D2 is the unsung hero. The AI is strong in him.\", \"It's weird how music makes everything more exciting like movies, tech videos, parties. It must tap into some deep, social part of the brain.\", 'I learned a new word yesterday. aliteracy: people who can read but choose not to.', 'Compared to the cold and hungry existence lived by our ancestors, our society is almost perfect. I tell myself that when I buy printer ink.', 'My daughter, age 3, kept asking me what time it was today. I guess she had somewhere to be.', \"The movie I watched about a tire that comes to life and starts killing people wasn't as good as I had hoped it would be.\", 'It would be nice to be able to cite videos in some permanent place.  Things disappear on YouTube.', \"Was just explaining @klout and how a person's influence is measured relative to the max of 100 held by Justin Bieber. This is our society.\", \"I've never been one to see symbolism in books or movies. I like to take fictional stories at face value, as if they were my own experience.\", 'Dreamed I played basketball with a giant strawberry for a ball. And there were South American soccer players camped out outside my window.', \"Why can't we get good disaster movies?\", \"Just got done working in the yard. We could solve the world's energy problems if we could somehow harness the power of cords to get tangled.\", 'Just saw Limitless. Great movie. But he said doubling your money every day was too slow. Clearly, even with NZT, he was still limited.', 'I love it when movies about ghosts are \"based on a true story.\"', 'Just watched Tyrannosaur. That movie breaks your heart over and over again. Fewer dinosaurs than I expected.', 'I love the smell of coffee in the morning. It smells like victory.', 'Sprinkling unexpected words into your prose can keep it fresh but can also distract the reader. #Trade-offs #Writing', 'Young people sometimes read the same books and watch the same movies multiple times. When we get older we stop doing that. Why?']\n",
      "Loss: 0.38\n",
      "Predictions: [1], Labels: [1], Accuracy: 1.0\n",
      "['I wonder why we raise our kids on movies with clear good guys and bad guys, when for the most part, the real world is just people']\n",
      "STARTING EPOCH:  5\n",
      "Loss: 0.58\n",
      "Predictions: [1 1 0 1 0 0 1 1 1 1 1 1 1 1 1 1 1 1 1 1], Labels: [0 1 0 0 0 0 0 1 1 0 1 1 1 1 1 0 1 0 0 0], Accuracy: 0.6\n",
      "['I was once at a party where someone spilled red wine, and they used white wine to clean it up. I wonder if the opposite would work.', \"I've largely cut sugar from my diet this week. Remember that Paul Giamatti movie Cold Souls? Me too.\", \"I wish my financial/retirement companies would stop sending me mail. With all these unnecessary words, I'm likely to miss important ones.\", \"Just got done working in the yard. We could solve the world's energy problems if we could somehow harness the power of cords to get tangled.\", \"I used to go one way to work. But lately, I've been asking Google Maps, and it sends me another way. I'm starting to see its wisdom.\", \"It's funny. In negotiations, the first person to specify a number loses, but the first person to establish a number as an anchor wins.\", \"Mute button didn't work on the box loudly squawking ads at the gas pump. Is the world not noisy enough? Those things are evil.\", 'It would be funny to make a movie where characters argue using only maxims. Early bird gets the worm! Second mouse gets the cheese!', \"One bummer about getting older is that I've already seen every new movie that comes out.\", \"Was just explaining @klout and how a person's influence is measured relative to the max of 100 held by Justin Bieber. This is our society.\", 'Remember that kid\\'s movie Escape from Planet Earth? One character had the coolest coffee cup ever. It said, \"I love safety.\" I want that.', 'Just saw The Muppets. Interesting movie about a small fringe group fighting to keep America dependent on foreign oil.', \"For picking movies, I've learned that not only must I enjoy it for 2 hours, but I must also enjoy having it in my head for days after.\", \"Some movies don't seem to have enough material to fill up the trailer. Don't know what they do for the other 1 hour and 28 minutes.\", 'Just saw Contagion. I thought it was pretty good. I like that it stayed with the big picture. Not as sappy as most disaster movies.', 'Amazing how black socks change to blue when you take them out of the dresser drawer.', \"There must be a special place in hell for the people who decided you can't skip movie previews on DVD.\", 'Want evidence of intelligent machines? Captchas are now too hard for me.', \"Hate wearing a belt, but I need to dress fancy for work. I'm waiting for elastic belts to come back in fashion.\", 'Office is out of coffee. Luckily, I have emergency backup. Kind of like the strategic petroleum reserve.']\n",
      "Loss: 0.47\n",
      "Predictions: [1 1 1 0 1 1 1 1 1 0 1 0 0 1 1 1 1 1 1 1], Labels: [1 1 1 0 1 1 0 0 1 0 1 0 0 0 1 1 1 1 1 0], Accuracy: 0.8\n",
      "['When they show scientists in movies, they never show them making PowerPoint slides.', 'I love it when movies about ghosts are \"based on a true story.\"', 'Love it when movies show you what it would be like to be someone else. Saw Last Train Home. Great movie about migrant workers in China.', 'Motto for the future of information: \"If it\\'s not relevant, get it out of my way.\"', 'Loved that movie Argo. It felt good to feel good about America, Hollywood, and Canada. #TwoThumbsUp', \"I've never been one to see symbolism in books or movies. I like to take fictional stories at face value, as if they were my own experience.\", 'Finally saw Napoleon Dynamite. Love how you can always hear the birds chirping in the background. And the steaks.', 'Sprinkling unexpected words into your prose can keep it fresh but can also distract the reader. #Trade-offs #Writing', 'Just saw Black Swan. Man, that was stressful. I need to relax, maybe watch a movie or something.', 'Sugar is the enemy. Sugar gives me life. #Productivity', 'One of my superpowers is the ability to abandon a book or movie halfway through.', 'Fame and fortune await the first person to develop a robot that powers itself by eating fire ants.', 'I love how a good book becomes part of your mental life forever.', \"I hate when I write a note to myself but then later don't remember what the note refers to.\", 'Someday, we are going to look back and be embarrassed about the number of comic book movies made during this time period.', \"I like a lot of kids movies, but I just can't get into these Ice Age films.\", 'Just got a wrong-number-call asking for a girl named Shirley. Always reminds me of that old movie Ruthless People.', 'Told my wife I rented a movie about an automobile tire that comes to life and murders people. She just walked out of the room. #Marriage', 'Just watched Tyrannosaur. That movie breaks your heart over and over again. Fewer dinosaurs than I expected.', 'If it\\'s \"not necessary to dial a 1 when calling this number\" can\\'t the phone company just figure it out? Google has spoiled us all.']\n",
      "Loss: 0.64\n",
      "Predictions: [0 1 1 1 0 1 1 0 1 1 1 0 0 0 1 1 1 1 1 0], Labels: [0 0 0 0 0 1 0 0 0 1 1 1 0 0 0 1 1 0 0 0], Accuracy: 0.55\n",
      "[\"I successfully resurrected my wife's old laptop by installing Ubuntu. My work here is done.\", 'I feel lucky to have a job where I learn every day. The world changes so fast--you have to run to keep up. #scary', 'Dreamed I played basketball with a giant strawberry for a ball. And there were South American soccer players camped out outside my window.', 'It would be nice to be able to cite videos in some permanent place.  Things disappear on YouTube.', 'Wrestling with my boys. Good to have a little cave-man time before I go back to work tomorrow.', \"With a big bucket of popcorn, it doesn't matter what the movie is. You're happy.\", 'Should the streets be widened to help emergency vehicles? Of course. Should my yard be shortened to widen the street? Get off my property.', 'I keep buying pyrite but I must be doing something wrong.', \"Hate when I can't find my mouse pointer. Even worse with two screens.\", 'Amazing how much of our time we spend in fantasy---movies, books, and TV. We live multiple simultaneous narratives.', \"A lot of my recurring dreams are like old movies that I don't quite remember. I know how they end, but I don't know how they'll get there.\", 'Movies should bring back quicksand. We all need to be reminded of the dangers.', 'My daughter, age 3, kept asking me what time it was today. I guess she had somewhere to be.', 'There is a tension in writing. You want to keep going to maintain momentum, but you also need to stop periodically to gain perspective.', \"My stomach is uncomfortably full, but I am still distracted by hunger. What's that story about the guy pushing the rock up the hill?\", 'Finally watched Dr. Strangelove. Great movie, but the last scene with the character Dr. Strangelove was ... stupid.', 'Went to Costco for the first time not too long ago. That place is a disaster movie.', \"Surprisingly, the search feature on Google docs doesn't work very well. If only they had access to search experts.\", 'My kids saw a typewriter today and were blown away. \"What IS that thing?\" Like it was from an alien world.', 'Network programming is somehow very satisfying. All of these happy little computers talking to each other.']\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Loss: 0.61\n",
      "Predictions: [1 1 0 0 1 1 1 1 1 1 1 0 1 0 1 1 1 1 0 1], Labels: [0 0 0 0 1 0 0 0 1 1 1 0 1 0 1 0 1 0 1 1], Accuracy: 0.6\n",
      "['I just wrote the word \"print\" and Google Docs flagged it as misspelled. It wanted me to write \"printf\". How cool is that?', 'I\\'ve been in the workforce for 15 years, and I still can\\'t get in the habit of saying \"good morning\" instead of \"hey\" or \"what\\'s up?\"', 'Science moves forward by more accurate predictions. Math by new theorems. How does philosophy move forward?', 'I love the smell of coffee in the morning. It smells like victory.', 'Recently watched the movie The Bay. Anyone want to go for a swim?', 'When younger, the soreness in my muscles after playing soccer was kind of cool. It meant I had worked hard. Now the soreness just hurts.', 'Always surprised people keep those email ad signatures, such as \"Sent from my iPhone.\" Consider changing to \"Sent from my subconscious.\"', 'I learned a new word yesterday. aliteracy: people who can read but choose not to.', 'Isn\\'t it weird that an actor is \"in\" a movie but \"on\" a TV show?', \"Finally saw Sleep Dealer (Traficante de Suenos). If you like dystopian movies about migrant works and drones, this one's for you. I loved it\", 'Young people sometimes read the same books and watch the same movies multiple times. When we get older we stop doing that. Why?', 'I\\'m used to \"twenty years ago\" meaning the 1970s, but that was twenty years ago.', 'Evil Dead was last horror movie I saw. Never seen that kind of movie before. The brain is not meant to process that kind of sensory input.', 'Compared to the cold and hungry existence lived by our ancestors, our society is almost perfect. I tell myself that when I buy printer ink.', \"Why can't we get good disaster movies?\", 'My inbox now has 0 messages. It feels good to get organized.', 'The weird thing about the Lord of the Rings movies is that they all seem to just end. Like they ran out of film or something. #TheHobbit', 'Good comedy consists of statements that are both true and absurd.', \"The movie I watched about a tire that comes to life and starts killing people wasn't as good as I had hoped it would be.\", 'Just saw Limitless. Great movie. But he said doubling your money every day was too slow. Clearly, even with NZT, he was still limited.']\n",
      "Loss: 0.71\n",
      "Predictions: [0 0 1 1 1 0 1 1 1 1 0 1 1 1 1 1 1 1 1 1], Labels: [0 0 1 0 0 0 0 0 0 0 1 0 1 0 1 0 1 1 1 1], Accuracy: 0.5\n",
      "['I coach kindergarten soccer. My standard April fools day joke is to email the parents about having two-a-day practices.', \"I hate when I have to get up early. That's enough, but my body will often get me up even earlier than that for no good reason.\", 'Talking about the 6 Star Wars movies with my kids. Terms like \"first one\" and \"last one\" result in limitless confusion.', \"Companies should realize that surveys must be short. I'm not going to click on  25 ovals, I just want to say the hotel carpet was dirty.\", 'It would be nice if we could abandon all of this inane talk about celebrities and go back to gossiping about people that we actually know.', \"According to toddlers, a cat's tail was made for pulling. The perfect Gibsonian affordance.\", 'I love getting reminders from Microsoft Outlook telling me that I am 23 hours late for a meeting.', \"It's funny how deep questions and ignorant questions look alike. E.g., what is science?\", \"It's amazing how much joy in life comes from making and executing plans.\", \"Last one in the office. I'm riding a scooter down an empty hall, and I'm suddenly reminded of the Big Wheel scene from The Shining.\", 'I love reading the reviews for kids movies. \"... the archetypes do not resonate as one would expect given the ...\" Kids want burp jokes.', 'Coat hangers are very poorly behaved objects.', \"Camping in the living room eating S'mores and watching a movie. It's surprising that they don't have more safety railings on the Death Star.\", \"Man, how do people watch TV? You're sitting there all happy, and then suddenly someone tries to sell you something. Over, and over again.\", \"After having watched many Stars Wars movies with my kids lately, I've decided that R2-D2 is the unsung hero. The AI is strong in him.\", \"My son (7) wants to know why he can't have a credit card.\", \"It's weird how music makes everything more exciting like movies, tech videos, parties. It must tap into some deep, social part of the brain.\", \"Recently watched Anna Karenina. That movie would make no sense at all if I hadn't read the book.\", 'I wonder why we raise our kids on movies with clear good guys and bad guys, when for the most part, the real world is just people', 'HT @TheOnion: 5-Year-Old Critics Agree: Movie \"Cars\" Only Gets Better After 40th Viewing']\n",
      "Loss: 0.93\n",
      "Predictions: [0], Labels: [1], Accuracy: 0.0\n",
      "[\"I bet that primitive people used to just sit around and tell stories about monsters. Anyway, I'm off to the movies.\"]\n",
      "STARTING EPOCH:  6\n",
      "Loss: 0.56\n",
      "Predictions: [0 1 1 0 0 0 1 1 1 0 0 1 1 0 0 0 0 0 1 1], Labels: [0 0 1 0 0 0 0 1 1 0 0 1 1 0 0 0 0 0 0 0], Accuracy: 0.8\n",
      "[\"I successfully resurrected my wife's old laptop by installing Ubuntu. My work here is done.\", 'Always surprised people keep those email ad signatures, such as \"Sent from my iPhone.\" Consider changing to \"Sent from my subconscious.\"', 'I love reading the reviews for kids movies. \"... the archetypes do not resonate as one would expect given the ...\" Kids want burp jokes.', 'Wrestling with my boys. Good to have a little cave-man time before I go back to work tomorrow.', 'It would be nice to be able to cite videos in some permanent place.  Things disappear on YouTube.', 'If it\\'s \"not necessary to dial a 1 when calling this number\" can\\'t the phone company just figure it out? Google has spoiled us all.', \"Man, how do people watch TV? You're sitting there all happy, and then suddenly someone tries to sell you something. Over, and over again.\", \"With a big bucket of popcorn, it doesn't matter what the movie is. You're happy.\", 'One of my superpowers is the ability to abandon a book or movie halfway through.', \"Hate wearing a belt, but I need to dress fancy for work. I'm waiting for elastic belts to come back in fashion.\", 'I love the smell of coffee in the morning. It smells like victory.', \"Finally saw Sleep Dealer (Traficante de Suenos). If you like dystopian movies about migrant works and drones, this one's for you. I loved it\", 'The weird thing about the Lord of the Rings movies is that they all seem to just end. Like they ran out of film or something. #TheHobbit', 'Should the streets be widened to help emergency vehicles? Of course. Should my yard be shortened to widen the street? Get off my property.', 'Office is out of coffee. Luckily, I have emergency backup. Kind of like the strategic petroleum reserve.', 'My kids saw a typewriter today and were blown away. \"What IS that thing?\" Like it was from an alien world.', \"I wish my financial/retirement companies would stop sending me mail. With all these unnecessary words, I'm likely to miss important ones.\", 'I\\'ve been in the workforce for 15 years, and I still can\\'t get in the habit of saying \"good morning\" instead of \"hey\" or \"what\\'s up?\"', 'Sugar is the enemy. Sugar gives me life. #Productivity', 'Good comedy consists of statements that are both true and absurd.']\n",
      "Loss: 0.48\n",
      "Predictions: [0 1 0 1 0 0 0 1 0 0 0 0 1 1 0 1 0 0 0 1], Labels: [0 1 0 1 0 0 0 1 0 0 0 1 0 0 0 1 0 0 0 1], Accuracy: 0.85\n",
      "[\"My son (7) wants to know why he can't have a credit card.\", 'Finally watched Dr. Strangelove. Great movie, but the last scene with the character Dr. Strangelove was ... stupid.', 'Want evidence of intelligent machines? Captchas are now too hard for me.', \"I've largely cut sugar from my diet this week. Remember that Paul Giamatti movie Cold Souls? Me too.\", 'I love how a good book becomes part of your mental life forever.', 'Compared to the cold and hungry existence lived by our ancestors, our society is almost perfect. I tell myself that when I buy printer ink.', 'Network programming is somehow very satisfying. All of these happy little computers talking to each other.', 'Isn\\'t it weird that an actor is \"in\" a movie but \"on\" a TV show?', \"Was just explaining @klout and how a person's influence is measured relative to the max of 100 held by Justin Bieber. This is our society.\", \"Last one in the office. I'm riding a scooter down an empty hall, and I'm suddenly reminded of the Big Wheel scene from The Shining.\", 'I\\'m used to \"twenty years ago\" meaning the 1970s, but that was twenty years ago.', 'Movies should bring back quicksand. We all need to be reminded of the dangers.', \"Just got done working in the yard. We could solve the world's energy problems if we could somehow harness the power of cords to get tangled.\", 'My daughter, age 3, kept asking me what time it was today. I guess she had somewhere to be.', \"Hate when I can't find my mouse pointer. Even worse with two screens.\", \"For picking movies, I've learned that not only must I enjoy it for 2 hours, but I must also enjoy having it in my head for days after.\", 'I coach kindergarten soccer. My standard April fools day joke is to email the parents about having two-a-day practices.', \"It's funny. In negotiations, the first person to specify a number loses, but the first person to establish a number as an anchor wins.\", 'When younger, the soreness in my muscles after playing soccer was kind of cool. It meant I had worked hard. Now the soreness just hurts.', 'Loved that movie Argo. It felt good to feel good about America, Hollywood, and Canada. #TwoThumbsUp']\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Loss: 0.42\n",
      "Predictions: [0 0 1 0 1 1 1 0 1 0 0 1 0 1 1 1 1 0 1 0], Labels: [0 0 1 0 1 1 1 0 1 0 0 1 0 0 1 0 1 0 1 0], Accuracy: 0.9\n",
      "['Science moves forward by more accurate predictions. Math by new theorems. How does philosophy move forward?', 'I love getting reminders from Microsoft Outlook telling me that I am 23 hours late for a meeting.', 'Went to Costco for the first time not too long ago. That place is a disaster movie.', 'Sprinkling unexpected words into your prose can keep it fresh but can also distract the reader. #Trade-offs #Writing', \"Camping in the living room eating S'mores and watching a movie. It's surprising that they don't have more safety railings on the Death Star.\", \"There must be a special place in hell for the people who decided you can't skip movie previews on DVD.\", 'I love it when movies about ghosts are \"based on a true story.\"', \"I used to go one way to work. But lately, I've been asking Google Maps, and it sends me another way. I'm starting to see its wisdom.\", 'Amazing how much of our time we spend in fantasy---movies, books, and TV. We live multiple simultaneous narratives.', \"My stomach is uncomfortably full, but I am still distracted by hunger. What's that story about the guy pushing the rock up the hill?\", \"I hate when I have to get up early. That's enough, but my body will often get me up even earlier than that for no good reason.\", 'Talking about the 6 Star Wars movies with my kids. Terms like \"first one\" and \"last one\" result in limitless confusion.', 'I learned a new word yesterday. aliteracy: people who can read but choose not to.', \"Mute button didn't work on the box loudly squawking ads at the gas pump. Is the world not noisy enough? Those things are evil.\", \"A lot of my recurring dreams are like old movies that I don't quite remember. I know how they end, but I don't know how they'll get there.\", 'I just wrote the word \"print\" and Google Docs flagged it as misspelled. It wanted me to write \"printf\". How cool is that?', 'When they show scientists in movies, they never show them making PowerPoint slides.', 'Dreamed I played basketball with a giant strawberry for a ball. And there were South American soccer players camped out outside my window.', \"The movie I watched about a tire that comes to life and starts killing people wasn't as good as I had hoped it would be.\", 'I keep buying pyrite but I must be doing something wrong.']\n",
      "Loss: 0.43\n",
      "Predictions: [1 1 0 0 1 0 0 1 1 0 0 0 1 0 1 1 1 0 1 1], Labels: [1 1 0 0 0 0 0 1 1 0 0 0 1 1 1 1 0 0 1 1], Accuracy: 0.85\n",
      "[\"After having watched many Stars Wars movies with my kids lately, I've decided that R2-D2 is the unsung hero. The AI is strong in him.\", 'Just saw Black Swan. Man, that was stressful. I need to relax, maybe watch a movie or something.', \"It's amazing how much joy in life comes from making and executing plans.\", \"Surprisingly, the search feature on Google docs doesn't work very well. If only they had access to search experts.\", \"According to toddlers, a cat's tail was made for pulling. The perfect Gibsonian affordance.\", 'I feel lucky to have a job where I learn every day. The world changes so fast--you have to run to keep up. #scary', 'It would be nice if we could abandon all of this inane talk about celebrities and go back to gossiping about people that we actually know.', 'Just got a wrong-number-call asking for a girl named Shirley. Always reminds me of that old movie Ruthless People.', 'Just saw The Muppets. Interesting movie about a small fringe group fighting to keep America dependent on foreign oil.', \"I hate when I write a note to myself but then later don't remember what the note refers to.\", 'Coat hangers are very poorly behaved objects.', 'I was once at a party where someone spilled red wine, and they used white wine to clean it up. I wonder if the opposite would work.', \"I like a lot of kids movies, but I just can't get into these Ice Age films.\", 'Young people sometimes read the same books and watch the same movies multiple times. When we get older we stop doing that. Why?', 'Just watched Tyrannosaur. That movie breaks your heart over and over again. Fewer dinosaurs than I expected.', 'Just saw Limitless. Great movie. But he said doubling your money every day was too slow. Clearly, even with NZT, he was still limited.', 'Amazing how black socks change to blue when you take them out of the dresser drawer.', \"It's funny how deep questions and ignorant questions look alike. E.g., what is science?\", 'Just saw Contagion. I thought it was pretty good. I like that it stayed with the big picture. Not as sappy as most disaster movies.', \"It's weird how music makes everything more exciting like movies, tech videos, parties. It must tap into some deep, social part of the brain.\"]\n",
      "Loss: 0.45\n",
      "Predictions: [1 1 1 1 0 1 0 1 1 1 1 0 1 0 0 0 0 0 1 0], Labels: [1 1 1 1 0 1 0 0 1 1 1 0 1 0 1 1 0 1 1 1], Accuracy: 0.75\n",
      "['It would be funny to make a movie where characters argue using only maxims. Early bird gets the worm! Second mouse gets the cheese!', 'Evil Dead was last horror movie I saw. Never seen that kind of movie before. The brain is not meant to process that kind of sensory input.', \"Recently watched Anna Karenina. That movie would make no sense at all if I hadn't read the book.\", 'Love it when movies show you what it would be like to be someone else. Saw Last Train Home. Great movie about migrant workers in China.', 'There is a tension in writing. You want to keep going to maintain momentum, but you also need to stop periodically to gain perspective.', \"One bummer about getting older is that I've already seen every new movie that comes out.\", 'Motto for the future of information: \"If it\\'s not relevant, get it out of my way.\"', 'Finally saw Napoleon Dynamite. Love how you can always hear the birds chirping in the background. And the steaks.', 'Someday, we are going to look back and be embarrassed about the number of comic book movies made during this time period.', 'Remember that kid\\'s movie Escape from Planet Earth? One character had the coolest coffee cup ever. It said, \"I love safety.\" I want that.', 'I wonder why we raise our kids on movies with clear good guys and bad guys, when for the most part, the real world is just people', 'My inbox now has 0 messages. It feels good to get organized.', 'HT @TheOnion: 5-Year-Old Critics Agree: Movie \"Cars\" Only Gets Better After 40th Viewing', \"Companies should realize that surveys must be short. I'm not going to click on  25 ovals, I just want to say the hotel carpet was dirty.\", 'Recently watched the movie The Bay. Anyone want to go for a swim?', \"I've never been one to see symbolism in books or movies. I like to take fictional stories at face value, as if they were my own experience.\", 'Fame and fortune await the first person to develop a robot that powers itself by eating fire ants.', \"I bet that primitive people used to just sit around and tell stories about monsters. Anyway, I'm off to the movies.\", 'Told my wife I rented a movie about an automobile tire that comes to life and murders people. She just walked out of the room. #Marriage', \"Why can't we get good disaster movies?\"]\n",
      "Loss: 0.20\n",
      "Predictions: [1], Labels: [1], Accuracy: 1.0\n",
      "[\"Some movies don't seem to have enough material to fill up the trailer. Don't know what they do for the other 1 hour and 28 minutes.\"]\n",
      "STARTING EPOCH:  7\n",
      "Loss: 0.40\n",
      "Predictions: [0 1 0 0 0 0 0 0 0 0 0 0 0 1 0 0 1 0 0 0], Labels: [0 1 0 1 1 0 1 0 0 0 0 0 0 1 0 0 1 0 0 1], Accuracy: 0.8\n",
      "['My kids saw a typewriter today and were blown away. \"What IS that thing?\" Like it was from an alien world.', 'Finally watched Dr. Strangelove. Great movie, but the last scene with the character Dr. Strangelove was ... stupid.', 'I\\'ve been in the workforce for 15 years, and I still can\\'t get in the habit of saying \"good morning\" instead of \"hey\" or \"what\\'s up?\"', 'Amazing how much of our time we spend in fantasy---movies, books, and TV. We live multiple simultaneous narratives.', 'Just watched Tyrannosaur. That movie breaks your heart over and over again. Fewer dinosaurs than I expected.', 'Wrestling with my boys. Good to have a little cave-man time before I go back to work tomorrow.', \"I've never been one to see symbolism in books or movies. I like to take fictional stories at face value, as if they were my own experience.\", \"I successfully resurrected my wife's old laptop by installing Ubuntu. My work here is done.\", 'Compared to the cold and hungry existence lived by our ancestors, our society is almost perfect. I tell myself that when I buy printer ink.', 'I feel lucky to have a job where I learn every day. The world changes so fast--you have to run to keep up. #scary', 'I learned a new word yesterday. aliteracy: people who can read but choose not to.', \"Man, how do people watch TV? You're sitting there all happy, and then suddenly someone tries to sell you something. Over, and over again.\", 'Science moves forward by more accurate predictions. Math by new theorems. How does philosophy move forward?', 'Just saw Limitless. Great movie. But he said doubling your money every day was too slow. Clearly, even with NZT, he was still limited.', \"I used to go one way to work. But lately, I've been asking Google Maps, and it sends me another way. I'm starting to see its wisdom.\", 'I love how a good book becomes part of your mental life forever.', 'Went to Costco for the first time not too long ago. That place is a disaster movie.', 'Fame and fortune await the first person to develop a robot that powers itself by eating fire ants.', \"It's amazing how much joy in life comes from making and executing plans.\", 'Young people sometimes read the same books and watch the same movies multiple times. When we get older we stop doing that. Why?']\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Loss: 0.43\n",
      "Predictions: [1 1 1 1 1 0 0 0 0 0 0 0 1 1 0 0 0 0 0 1], Labels: [1 1 1 0 1 0 1 0 0 0 0 0 1 0 0 0 0 0 0 1], Accuracy: 0.85\n",
      "['I wonder why we raise our kids on movies with clear good guys and bad guys, when for the most part, the real world is just people', 'Just saw Black Swan. Man, that was stressful. I need to relax, maybe watch a movie or something.', \"For picking movies, I've learned that not only must I enjoy it for 2 hours, but I must also enjoy having it in my head for days after.\", 'I just wrote the word \"print\" and Google Docs flagged it as misspelled. It wanted me to write \"printf\". How cool is that?', 'One of my superpowers is the ability to abandon a book or movie halfway through.', 'Should the streets be widened to help emergency vehicles? Of course. Should my yard be shortened to widen the street? Get off my property.', 'Just saw Contagion. I thought it was pretty good. I like that it stayed with the big picture. Not as sappy as most disaster movies.', 'Coat hangers are very poorly behaved objects.', 'Network programming is somehow very satisfying. All of these happy little computers talking to each other.', 'My inbox now has 0 messages. It feels good to get organized.', \"Hate wearing a belt, but I need to dress fancy for work. I'm waiting for elastic belts to come back in fashion.\", 'Good comedy consists of statements that are both true and absurd.', \"It's weird how music makes everything more exciting like movies, tech videos, parties. It must tap into some deep, social part of the brain.\", \"Surprisingly, the search feature on Google docs doesn't work very well. If only they had access to search experts.\", 'Sprinkling unexpected words into your prose can keep it fresh but can also distract the reader. #Trade-offs #Writing', 'When younger, the soreness in my muscles after playing soccer was kind of cool. It meant I had worked hard. Now the soreness just hurts.', \"It's funny. In negotiations, the first person to specify a number loses, but the first person to establish a number as an anchor wins.\", \"Mute button didn't work on the box loudly squawking ads at the gas pump. Is the world not noisy enough? Those things are evil.\", \"It's funny how deep questions and ignorant questions look alike. E.g., what is science?\", \"Finally saw Sleep Dealer (Traficante de Suenos). If you like dystopian movies about migrant works and drones, this one's for you. I loved it\"]\n",
      "Loss: 0.44\n",
      "Predictions: [1 1 0 1 1 1 0 0 1 1 0 1 0 0 0 0 0 0 0 1], Labels: [1 1 0 1 1 1 0 0 1 1 1 1 1 0 1 0 1 0 0 0], Accuracy: 0.75\n",
      "['Just got a wrong-number-call asking for a girl named Shirley. Always reminds me of that old movie Ruthless People.', 'Someday, we are going to look back and be embarrassed about the number of comic book movies made during this time period.', 'Want evidence of intelligent machines? Captchas are now too hard for me.', 'Loved that movie Argo. It felt good to feel good about America, Hollywood, and Canada. #TwoThumbsUp', 'When they show scientists in movies, they never show them making PowerPoint slides.', \"Why can't we get good disaster movies?\", \"My stomach is uncomfortably full, but I am still distracted by hunger. What's that story about the guy pushing the rock up the hill?\", 'Dreamed I played basketball with a giant strawberry for a ball. And there were South American soccer players camped out outside my window.', 'Talking about the 6 Star Wars movies with my kids. Terms like \"first one\" and \"last one\" result in limitless confusion.', \"With a big bucket of popcorn, it doesn't matter what the movie is. You're happy.\", 'I love it when movies about ghosts are \"based on a true story.\"', 'The weird thing about the Lord of the Rings movies is that they all seem to just end. Like they ran out of film or something. #TheHobbit', 'It would be funny to make a movie where characters argue using only maxims. Early bird gets the worm! Second mouse gets the cheese!', 'I love getting reminders from Microsoft Outlook telling me that I am 23 hours late for a meeting.', 'Movies should bring back quicksand. We all need to be reminded of the dangers.', 'I\\'m used to \"twenty years ago\" meaning the 1970s, but that was twenty years ago.', 'I love reading the reviews for kids movies. \"... the archetypes do not resonate as one would expect given the ...\" Kids want burp jokes.', 'I love the smell of coffee in the morning. It smells like victory.', \"I hate when I have to get up early. That's enough, but my body will often get me up even earlier than that for no good reason.\", \"Just got done working in the yard. We could solve the world's energy problems if we could somehow harness the power of cords to get tangled.\"]\n",
      "Loss: 0.33\n",
      "Predictions: [0 0 1 1 1 1 0 1 1 0 0 0 1 1 1 0 0 1 1 1], Labels: [0 0 1 1 1 1 0 0 1 0 0 0 1 1 1 0 0 1 1 1], Accuracy: 0.95\n",
      "[\"My son (7) wants to know why he can't have a credit card.\", 'My daughter, age 3, kept asking me what time it was today. I guess she had somewhere to be.', 'Recently watched the movie The Bay. Anyone want to go for a swim?', \"Recently watched Anna Karenina. That movie would make no sense at all if I hadn't read the book.\", 'Evil Dead was last horror movie I saw. Never seen that kind of movie before. The brain is not meant to process that kind of sensory input.', 'HT @TheOnion: 5-Year-Old Critics Agree: Movie \"Cars\" Only Gets Better After 40th Viewing', 'Always surprised people keep those email ad signatures, such as \"Sent from my iPhone.\" Consider changing to \"Sent from my subconscious.\"', 'Finally saw Napoleon Dynamite. Love how you can always hear the birds chirping in the background. And the steaks.', 'Just saw The Muppets. Interesting movie about a small fringe group fighting to keep America dependent on foreign oil.', 'It would be nice if we could abandon all of this inane talk about celebrities and go back to gossiping about people that we actually know.', \"Last one in the office. I'm riding a scooter down an empty hall, and I'm suddenly reminded of the Big Wheel scene from The Shining.\", 'I coach kindergarten soccer. My standard April fools day joke is to email the parents about having two-a-day practices.', \"One bummer about getting older is that I've already seen every new movie that comes out.\", \"I bet that primitive people used to just sit around and tell stories about monsters. Anyway, I'm off to the movies.\", \"After having watched many Stars Wars movies with my kids lately, I've decided that R2-D2 is the unsung hero. The AI is strong in him.\", \"According to toddlers, a cat's tail was made for pulling. The perfect Gibsonian affordance.\", \"Hate when I can't find my mouse pointer. Even worse with two screens.\", \"There must be a special place in hell for the people who decided you can't skip movie previews on DVD.\", 'Told my wife I rented a movie about an automobile tire that comes to life and murders people. She just walked out of the room. #Marriage', 'Love it when movies show you what it would be like to be someone else. Saw Last Train Home. Great movie about migrant workers in China.']\n",
      "Loss: 0.26\n",
      "Predictions: [0 0 1 1 0 1 0 0 0 0 0 0 0 0 0 1 0 1 0 1], Labels: [0 0 1 1 0 1 0 0 0 0 0 0 1 0 0 1 0 1 0 1], Accuracy: 0.95\n",
      "['Office is out of coffee. Luckily, I have emergency backup. Kind of like the strategic petroleum reserve.', \"I hate when I write a note to myself but then later don't remember what the note refers to.\", \"A lot of my recurring dreams are like old movies that I don't quite remember. I know how they end, but I don't know how they'll get there.\", \"Camping in the living room eating S'mores and watching a movie. It's surprising that they don't have more safety railings on the Death Star.\", 'There is a tension in writing. You want to keep going to maintain momentum, but you also need to stop periodically to gain perspective.', \"The movie I watched about a tire that comes to life and starts killing people wasn't as good as I had hoped it would be.\", 'If it\\'s \"not necessary to dial a 1 when calling this number\" can\\'t the phone company just figure it out? Google has spoiled us all.', 'Sugar is the enemy. Sugar gives me life. #Productivity', 'Amazing how black socks change to blue when you take them out of the dresser drawer.', 'Motto for the future of information: \"If it\\'s not relevant, get it out of my way.\"', 'I was once at a party where someone spilled red wine, and they used white wine to clean it up. I wonder if the opposite would work.', 'It would be nice to be able to cite videos in some permanent place.  Things disappear on YouTube.', \"I've largely cut sugar from my diet this week. Remember that Paul Giamatti movie Cold Souls? Me too.\", 'I keep buying pyrite but I must be doing something wrong.', \"Companies should realize that surveys must be short. I'm not going to click on  25 ovals, I just want to say the hotel carpet was dirty.\", \"I like a lot of kids movies, but I just can't get into these Ice Age films.\", \"Was just explaining @klout and how a person's influence is measured relative to the max of 100 held by Justin Bieber. This is our society.\", 'Remember that kid\\'s movie Escape from Planet Earth? One character had the coolest coffee cup ever. It said, \"I love safety.\" I want that.', \"I wish my financial/retirement companies would stop sending me mail. With all these unnecessary words, I'm likely to miss important ones.\", 'Isn\\'t it weird that an actor is \"in\" a movie but \"on\" a TV show?']\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Loss: 0.17\n",
      "Predictions: [1], Labels: [1], Accuracy: 1.0\n",
      "[\"Some movies don't seem to have enough material to fill up the trailer. Don't know what they do for the other 1 hour and 28 minutes.\"]\n",
      "STARTING EPOCH:  8\n",
      "Loss: 0.28\n",
      "Predictions: [1 0 1 1 0 1 0 1 0 0 0 0 0 1 0 1 0 1 0 1], Labels: [1 0 0 1 0 0 0 1 0 0 0 0 0 1 0 1 0 1 0 1], Accuracy: 0.9\n",
      "[\"There must be a special place in hell for the people who decided you can't skip movie previews on DVD.\", \"My stomach is uncomfortably full, but I am still distracted by hunger. What's that story about the guy pushing the rock up the hill?\", 'Wrestling with my boys. Good to have a little cave-man time before I go back to work tomorrow.', 'I love reading the reviews for kids movies. \"... the archetypes do not resonate as one would expect given the ...\" Kids want burp jokes.', \"I hate when I have to get up early. That's enough, but my body will often get me up even earlier than that for no good reason.\", \"It's amazing how much joy in life comes from making and executing plans.\", 'Fame and fortune await the first person to develop a robot that powers itself by eating fire ants.', 'Just saw Contagion. I thought it was pretty good. I like that it stayed with the big picture. Not as sappy as most disaster movies.', 'Coat hangers are very poorly behaved objects.', \"Man, how do people watch TV? You're sitting there all happy, and then suddenly someone tries to sell you something. Over, and over again.\", \"It's funny. In negotiations, the first person to specify a number loses, but the first person to establish a number as an anchor wins.\", \"Hate wearing a belt, but I need to dress fancy for work. I'm waiting for elastic belts to come back in fashion.\", \"Surprisingly, the search feature on Google docs doesn't work very well. If only they had access to search experts.\", \"I like a lot of kids movies, but I just can't get into these Ice Age films.\", \"I successfully resurrected my wife's old laptop by installing Ubuntu. My work here is done.\", \"Finally saw Sleep Dealer (Traficante de Suenos). If you like dystopian movies about migrant works and drones, this one's for you. I loved it\", 'Good comedy consists of statements that are both true and absurd.', 'Isn\\'t it weird that an actor is \"in\" a movie but \"on\" a TV show?', \"Hate when I can't find my mouse pointer. Even worse with two screens.\", \"For picking movies, I've learned that not only must I enjoy it for 2 hours, but I must also enjoy having it in my head for days after.\"]\n",
      "Loss: 0.22\n",
      "Predictions: [0 0 0 1 0 0 0 1 1 0 0 1 1 0 0 1 0 0 0 1], Labels: [0 0 0 1 0 0 0 1 1 1 0 1 1 0 0 1 0 0 0 1], Accuracy: 0.95\n",
      "['Should the streets be widened to help emergency vehicles? Of course. Should my yard be shortened to widen the street? Get off my property.', \"I used to go one way to work. But lately, I've been asking Google Maps, and it sends me another way. I'm starting to see its wisdom.\", 'I coach kindergarten soccer. My standard April fools day joke is to email the parents about having two-a-day practices.', 'It would be funny to make a movie where characters argue using only maxims. Early bird gets the worm! Second mouse gets the cheese!', 'Finally saw Napoleon Dynamite. Love how you can always hear the birds chirping in the background. And the steaks.', 'I\\'m used to \"twenty years ago\" meaning the 1970s, but that was twenty years ago.', 'I keep buying pyrite but I must be doing something wrong.', 'The weird thing about the Lord of the Rings movies is that they all seem to just end. Like they ran out of film or something. #TheHobbit', 'Someday, we are going to look back and be embarrassed about the number of comic book movies made during this time period.', \"The movie I watched about a tire that comes to life and starts killing people wasn't as good as I had hoped it would be.\", 'I love how a good book becomes part of your mental life forever.', 'Remember that kid\\'s movie Escape from Planet Earth? One character had the coolest coffee cup ever. It said, \"I love safety.\" I want that.', \"With a big bucket of popcorn, it doesn't matter what the movie is. You're happy.\", 'Office is out of coffee. Luckily, I have emergency backup. Kind of like the strategic petroleum reserve.', 'I feel lucky to have a job where I learn every day. The world changes so fast--you have to run to keep up. #scary', \"Some movies don't seem to have enough material to fill up the trailer. Don't know what they do for the other 1 hour and 28 minutes.\", 'Science moves forward by more accurate predictions. Math by new theorems. How does philosophy move forward?', 'Sugar is the enemy. Sugar gives me life. #Productivity', 'If it\\'s \"not necessary to dial a 1 when calling this number\" can\\'t the phone company just figure it out? Google has spoiled us all.', 'When they show scientists in movies, they never show them making PowerPoint slides.']\n",
      "Loss: 0.45\n",
      "Predictions: [0 1 0 0 1 1 1 0 0 0 0 0 0 0 1 1 1 1 0 1], Labels: [0 0 0 0 1 1 1 0 1 1 0 0 0 0 1 1 1 0 0 1], Accuracy: 0.8\n",
      "[\"It's funny how deep questions and ignorant questions look alike. E.g., what is science?\", 'It would be nice if we could abandon all of this inane talk about celebrities and go back to gossiping about people that we actually know.', \"Just got done working in the yard. We could solve the world's energy problems if we could somehow harness the power of cords to get tangled.\", \"I wish my financial/retirement companies would stop sending me mail. With all these unnecessary words, I'm likely to miss important ones.\", 'Just got a wrong-number-call asking for a girl named Shirley. Always reminds me of that old movie Ruthless People.', \"A lot of my recurring dreams are like old movies that I don't quite remember. I know how they end, but I don't know how they'll get there.\", \"I bet that primitive people used to just sit around and tell stories about monsters. Anyway, I'm off to the movies.\", \"Was just explaining @klout and how a person's influence is measured relative to the max of 100 held by Justin Bieber. This is our society.\", \"I've never been one to see symbolism in books or movies. I like to take fictional stories at face value, as if they were my own experience.\", 'Movies should bring back quicksand. We all need to be reminded of the dangers.', 'Motto for the future of information: \"If it\\'s not relevant, get it out of my way.\"', 'I just wrote the word \"print\" and Google Docs flagged it as misspelled. It wanted me to write \"printf\". How cool is that?', \"My son (7) wants to know why he can't have a credit card.\", 'I learned a new word yesterday. aliteracy: people who can read but choose not to.', \"Recently watched Anna Karenina. That movie would make no sense at all if I hadn't read the book.\", \"I've largely cut sugar from my diet this week. Remember that Paul Giamatti movie Cold Souls? Me too.\", 'Went to Costco for the first time not too long ago. That place is a disaster movie.', 'My kids saw a typewriter today and were blown away. \"What IS that thing?\" Like it was from an alien world.', 'Sprinkling unexpected words into your prose can keep it fresh but can also distract the reader. #Trade-offs #Writing', 'I love it when movies about ghosts are \"based on a true story.\"']\n",
      "Loss: 0.34\n",
      "Predictions: [0 1 1 1 0 1 0 0 1 0 1 1 1 0 0 1 1 0 1 1], Labels: [0 1 1 1 0 0 0 1 1 0 1 1 1 0 0 1 1 0 1 0], Accuracy: 0.85\n",
      "['My inbox now has 0 messages. It feels good to get organized.', 'Just saw Black Swan. Man, that was stressful. I need to relax, maybe watch a movie or something.', 'HT @TheOnion: 5-Year-Old Critics Agree: Movie \"Cars\" Only Gets Better After 40th Viewing', \"It's weird how music makes everything more exciting like movies, tech videos, parties. It must tap into some deep, social part of the brain.\", \"I hate when I write a note to myself but then later don't remember what the note refers to.\", 'When younger, the soreness in my muscles after playing soccer was kind of cool. It meant I had worked hard. Now the soreness just hurts.', 'Always surprised people keep those email ad signatures, such as \"Sent from my iPhone.\" Consider changing to \"Sent from my subconscious.\"', 'Amazing how much of our time we spend in fantasy---movies, books, and TV. We live multiple simultaneous narratives.', 'Just watched Tyrannosaur. That movie breaks your heart over and over again. Fewer dinosaurs than I expected.', 'I was once at a party where someone spilled red wine, and they used white wine to clean it up. I wonder if the opposite would work.', \"Camping in the living room eating S'mores and watching a movie. It's surprising that they don't have more safety railings on the Death Star.\", 'Evil Dead was last horror movie I saw. Never seen that kind of movie before. The brain is not meant to process that kind of sensory input.', \"Why can't we get good disaster movies?\", \"Mute button didn't work on the box loudly squawking ads at the gas pump. Is the world not noisy enough? Those things are evil.\", 'I love getting reminders from Microsoft Outlook telling me that I am 23 hours late for a meeting.', 'Talking about the 6 Star Wars movies with my kids. Terms like \"first one\" and \"last one\" result in limitless confusion.', 'Told my wife I rented a movie about an automobile tire that comes to life and murders people. She just walked out of the room. #Marriage', 'I love the smell of coffee in the morning. It smells like victory.', \"After having watched many Stars Wars movies with my kids lately, I've decided that R2-D2 is the unsung hero. The AI is strong in him.\", 'My daughter, age 3, kept asking me what time it was today. I guess she had somewhere to be.']\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Loss: 0.28\n",
      "Predictions: [0 1 1 1 0 0 0 0 1 0 0 1 1 0 0 0 1 0 0 1], Labels: [0 1 1 1 0 1 1 0 1 0 0 1 1 0 0 0 1 0 0 1], Accuracy: 0.9\n",
      "['I\\'ve been in the workforce for 15 years, and I still can\\'t get in the habit of saying \"good morning\" instead of \"hey\" or \"what\\'s up?\"', 'I wonder why we raise our kids on movies with clear good guys and bad guys, when for the most part, the real world is just people', 'Just saw Limitless. Great movie. But he said doubling your money every day was too slow. Clearly, even with NZT, he was still limited.', 'One of my superpowers is the ability to abandon a book or movie halfway through.', 'Compared to the cold and hungry existence lived by our ancestors, our society is almost perfect. I tell myself that when I buy printer ink.', 'Recently watched the movie The Bay. Anyone want to go for a swim?', 'Young people sometimes read the same books and watch the same movies multiple times. When we get older we stop doing that. Why?', \"Last one in the office. I'm riding a scooter down an empty hall, and I'm suddenly reminded of the Big Wheel scene from The Shining.\", 'Love it when movies show you what it would be like to be someone else. Saw Last Train Home. Great movie about migrant workers in China.', 'It would be nice to be able to cite videos in some permanent place.  Things disappear on YouTube.', 'Dreamed I played basketball with a giant strawberry for a ball. And there were South American soccer players camped out outside my window.', 'Finally watched Dr. Strangelove. Great movie, but the last scene with the character Dr. Strangelove was ... stupid.', 'Loved that movie Argo. It felt good to feel good about America, Hollywood, and Canada. #TwoThumbsUp', \"According to toddlers, a cat's tail was made for pulling. The perfect Gibsonian affordance.\", 'Want evidence of intelligent machines? Captchas are now too hard for me.', \"Companies should realize that surveys must be short. I'm not going to click on  25 ovals, I just want to say the hotel carpet was dirty.\", \"One bummer about getting older is that I've already seen every new movie that comes out.\", 'There is a tension in writing. You want to keep going to maintain momentum, but you also need to stop periodically to gain perspective.', 'Network programming is somehow very satisfying. All of these happy little computers talking to each other.', 'Just saw The Muppets. Interesting movie about a small fringe group fighting to keep America dependent on foreign oil.']\n",
      "Loss: 0.08\n",
      "Predictions: [0], Labels: [0], Accuracy: 1.0\n",
      "['Amazing how black socks change to blue when you take them out of the dresser drawer.']\n",
      "STARTING EPOCH:  9\n",
      "Loss: 0.18\n",
      "Predictions: [0 1 1 1 1 0 0 0 0 1 0 0 0 1 1 0 1 0 0 0], Labels: [0 1 1 1 1 0 0 0 0 1 0 0 0 0 1 0 1 0 0 0], Accuracy: 0.95\n",
      "[\"Hate when I can't find my mouse pointer. Even worse with two screens.\", \"I've largely cut sugar from my diet this week. Remember that Paul Giamatti movie Cold Souls? Me too.\", 'I love reading the reviews for kids movies. \"... the archetypes do not resonate as one would expect given the ...\" Kids want burp jokes.', 'Remember that kid\\'s movie Escape from Planet Earth? One character had the coolest coffee cup ever. It said, \"I love safety.\" I want that.', \"I bet that primitive people used to just sit around and tell stories about monsters. Anyway, I'm off to the movies.\", \"Was just explaining @klout and how a person's influence is measured relative to the max of 100 held by Justin Bieber. This is our society.\", \"It's amazing how much joy in life comes from making and executing plans.\", 'When younger, the soreness in my muscles after playing soccer was kind of cool. It meant I had worked hard. Now the soreness just hurts.', 'Want evidence of intelligent machines? Captchas are now too hard for me.', 'Just saw Limitless. Great movie. But he said doubling your money every day was too slow. Clearly, even with NZT, he was still limited.', 'I just wrote the word \"print\" and Google Docs flagged it as misspelled. It wanted me to write \"printf\". How cool is that?', 'Motto for the future of information: \"If it\\'s not relevant, get it out of my way.\"', 'Compared to the cold and hungry existence lived by our ancestors, our society is almost perfect. I tell myself that when I buy printer ink.', \"My stomach is uncomfortably full, but I am still distracted by hunger. What's that story about the guy pushing the rock up the hill?\", 'Just saw Contagion. I thought it was pretty good. I like that it stayed with the big picture. Not as sappy as most disaster movies.', 'I\\'m used to \"twenty years ago\" meaning the 1970s, but that was twenty years ago.', 'It would be funny to make a movie where characters argue using only maxims. Early bird gets the worm! Second mouse gets the cheese!', \"Mute button didn't work on the box loudly squawking ads at the gas pump. Is the world not noisy enough? Those things are evil.\", 'It would be nice to be able to cite videos in some permanent place.  Things disappear on YouTube.', 'Network programming is somehow very satisfying. All of these happy little computers talking to each other.']\n",
      "Loss: 0.25\n",
      "Predictions: [1 1 0 0 1 0 0 1 0 0 1 0 0 1 1 0 0 0 0 0], Labels: [1 1 0 0 1 0 0 1 0 0 1 0 1 1 1 0 0 0 1 0], Accuracy: 0.9\n",
      "[\"Some movies don't seem to have enough material to fill up the trailer. Don't know what they do for the other 1 hour and 28 minutes.\", 'The weird thing about the Lord of the Rings movies is that they all seem to just end. Like they ran out of film or something. #TheHobbit', 'Wrestling with my boys. Good to have a little cave-man time before I go back to work tomorrow.', 'Always surprised people keep those email ad signatures, such as \"Sent from my iPhone.\" Consider changing to \"Sent from my subconscious.\"', \"One bummer about getting older is that I've already seen every new movie that comes out.\", 'My kids saw a typewriter today and were blown away. \"What IS that thing?\" Like it was from an alien world.', 'I love getting reminders from Microsoft Outlook telling me that I am 23 hours late for a meeting.', 'I wonder why we raise our kids on movies with clear good guys and bad guys, when for the most part, the real world is just people', 'Good comedy consists of statements that are both true and absurd.', \"I hate when I have to get up early. That's enough, but my body will often get me up even earlier than that for no good reason.\", 'Just saw The Muppets. Interesting movie about a small fringe group fighting to keep America dependent on foreign oil.', 'Office is out of coffee. Luckily, I have emergency backup. Kind of like the strategic petroleum reserve.', \"Why can't we get good disaster movies?\", 'Love it when movies show you what it would be like to be someone else. Saw Last Train Home. Great movie about migrant workers in China.', 'Recently watched the movie The Bay. Anyone want to go for a swim?', 'I\\'ve been in the workforce for 15 years, and I still can\\'t get in the habit of saying \"good morning\" instead of \"hey\" or \"what\\'s up?\"', 'I learned a new word yesterday. aliteracy: people who can read but choose not to.', 'It would be nice if we could abandon all of this inane talk about celebrities and go back to gossiping about people that we actually know.', 'I love it when movies about ghosts are \"based on a true story.\"', \"I wish my financial/retirement companies would stop sending me mail. With all these unnecessary words, I'm likely to miss important ones.\"]\n",
      "Loss: 0.27\n",
      "Predictions: [1 0 0 0 0 0 0 0 0 1 1 0 1 0 1 0 1 0 0 1], Labels: [1 0 0 0 1 0 0 1 0 1 1 0 1 1 1 0 1 1 0 0], Accuracy: 0.75\n",
      "['Just got a wrong-number-call asking for a girl named Shirley. Always reminds me of that old movie Ruthless People.', 'Sprinkling unexpected words into your prose can keep it fresh but can also distract the reader. #Trade-offs #Writing', 'Science moves forward by more accurate predictions. Math by new theorems. How does philosophy move forward?', 'Finally saw Napoleon Dynamite. Love how you can always hear the birds chirping in the background. And the steaks.', 'Amazing how much of our time we spend in fantasy---movies, books, and TV. We live multiple simultaneous narratives.', 'I love the smell of coffee in the morning. It smells like victory.', 'I was once at a party where someone spilled red wine, and they used white wine to clean it up. I wonder if the opposite would work.', 'Isn\\'t it weird that an actor is \"in\" a movie but \"on\" a TV show?', 'Should the streets be widened to help emergency vehicles? Of course. Should my yard be shortened to widen the street? Get off my property.', \"After having watched many Stars Wars movies with my kids lately, I've decided that R2-D2 is the unsung hero. The AI is strong in him.\", \"Finally saw Sleep Dealer (Traficante de Suenos). If you like dystopian movies about migrant works and drones, this one's for you. I loved it\", \"It's funny. In negotiations, the first person to specify a number loses, but the first person to establish a number as an anchor wins.\", \"Recently watched Anna Karenina. That movie would make no sense at all if I hadn't read the book.\", 'Loved that movie Argo. It felt good to feel good about America, Hollywood, and Canada. #TwoThumbsUp', \"A lot of my recurring dreams are like old movies that I don't quite remember. I know how they end, but I don't know how they'll get there.\", \"Surprisingly, the search feature on Google docs doesn't work very well. If only they had access to search experts.\", \"I like a lot of kids movies, but I just can't get into these Ice Age films.\", \"It's weird how music makes everything more exciting like movies, tech videos, parties. It must tap into some deep, social part of the brain.\", \"It's funny how deep questions and ignorant questions look alike. E.g., what is science?\", \"Just got done working in the yard. We could solve the world's energy problems if we could somehow harness the power of cords to get tangled.\"]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Loss: 0.20\n",
      "Predictions: [0 0 1 0 0 1 1 0 1 1 0 1 1 0 0 0 0 1 1 0], Labels: [0 1 1 1 0 1 1 0 1 1 0 1 1 0 0 0 0 1 1 0], Accuracy: 0.9\n",
      "[\"My son (7) wants to know why he can't have a credit card.\", \"I've never been one to see symbolism in books or movies. I like to take fictional stories at face value, as if they were my own experience.\", 'Talking about the 6 Star Wars movies with my kids. Terms like \"first one\" and \"last one\" result in limitless confusion.', 'When they show scientists in movies, they never show them making PowerPoint slides.', 'I love how a good book becomes part of your mental life forever.', \"With a big bucket of popcorn, it doesn't matter what the movie is. You're happy.\", 'HT @TheOnion: 5-Year-Old Critics Agree: Movie \"Cars\" Only Gets Better After 40th Viewing', \"Last one in the office. I'm riding a scooter down an empty hall, and I'm suddenly reminded of the Big Wheel scene from The Shining.\", 'Went to Costco for the first time not too long ago. That place is a disaster movie.', 'Someday, we are going to look back and be embarrassed about the number of comic book movies made during this time period.', 'My daughter, age 3, kept asking me what time it was today. I guess she had somewhere to be.', 'Evil Dead was last horror movie I saw. Never seen that kind of movie before. The brain is not meant to process that kind of sensory input.', 'Just watched Tyrannosaur. That movie breaks your heart over and over again. Fewer dinosaurs than I expected.', \"I successfully resurrected my wife's old laptop by installing Ubuntu. My work here is done.\", 'There is a tension in writing. You want to keep going to maintain momentum, but you also need to stop periodically to gain perspective.', \"Man, how do people watch TV? You're sitting there all happy, and then suddenly someone tries to sell you something. Over, and over again.\", 'I feel lucky to have a job where I learn every day. The world changes so fast--you have to run to keep up. #scary', 'One of my superpowers is the ability to abandon a book or movie halfway through.', \"For picking movies, I've learned that not only must I enjoy it for 2 hours, but I must also enjoy having it in my head for days after.\", 'I keep buying pyrite but I must be doing something wrong.']\n",
      "Loss: 0.30\n",
      "Predictions: [1 0 1 0 1 0 0 0 0 1 0 0 1 0 1 0 1 1 0 0], Labels: [1 0 1 0 0 0 0 0 0 1 0 0 1 0 0 1 1 1 0 0], Accuracy: 0.85\n",
      "['Finally watched Dr. Strangelove. Great movie, but the last scene with the character Dr. Strangelove was ... stupid.', 'My inbox now has 0 messages. It feels good to get organized.', \"The movie I watched about a tire that comes to life and starts killing people wasn't as good as I had hoped it would be.\", \"According to toddlers, a cat's tail was made for pulling. The perfect Gibsonian affordance.\", \"I hate when I write a note to myself but then later don't remember what the note refers to.\", \"I used to go one way to work. But lately, I've been asking Google Maps, and it sends me another way. I'm starting to see its wisdom.\", 'Dreamed I played basketball with a giant strawberry for a ball. And there were South American soccer players camped out outside my window.', 'Amazing how black socks change to blue when you take them out of the dresser drawer.', 'Fame and fortune await the first person to develop a robot that powers itself by eating fire ants.', \"There must be a special place in hell for the people who decided you can't skip movie previews on DVD.\", 'Coat hangers are very poorly behaved objects.', 'If it\\'s \"not necessary to dial a 1 when calling this number\" can\\'t the phone company just figure it out? Google has spoiled us all.', 'Told my wife I rented a movie about an automobile tire that comes to life and murders people. She just walked out of the room. #Marriage', 'Sugar is the enemy. Sugar gives me life. #Productivity', \"Hate wearing a belt, but I need to dress fancy for work. I'm waiting for elastic belts to come back in fashion.\", 'Movies should bring back quicksand. We all need to be reminded of the dangers.', \"Camping in the living room eating S'mores and watching a movie. It's surprising that they don't have more safety railings on the Death Star.\", 'Young people sometimes read the same books and watch the same movies multiple times. When we get older we stop doing that. Why?', 'I coach kindergarten soccer. My standard April fools day joke is to email the parents about having two-a-day practices.', \"Companies should realize that surveys must be short. I'm not going to click on  25 ovals, I just want to say the hotel carpet was dirty.\"]\n",
      "Loss: 0.01\n",
      "Predictions: [1], Labels: [1], Accuracy: 1.0\n",
      "['Just saw Black Swan. Man, that was stressful. I need to relax, maybe watch a movie or something.']\n"
     ]
    }
   ],
   "source": [
    "# For writing to TensorBoard\n",
    "summary = SummaryWriter(data_dir)\n",
    "\n",
    "global_step = 0\n",
    "\n",
    "for epoch in range(10):\n",
    "\n",
    "    print(\"STARTING EPOCH: \",epoch)\n",
    "\n",
    "    for input_tuple in train_data_loader:\n",
    "\n",
    "        global_step += 1\n",
    "        input_data, mask_data, labels, tweets = input_tuple\n",
    "        log_softmax, _vectors = classifier(input_data, mask_data)\n",
    "        loss = F.nll_loss(log_softmax, labels)\n",
    "\n",
    "        # Gradients accumulate and you have to clear them out ever time\n",
    "        bert_optimizer.zero_grad()\n",
    "        head_optimizer.zero_grad()\n",
    "\n",
    "        # Get the gradients with respect to the parameters\n",
    "        loss.backward()\n",
    "\n",
    "        # Update the parameters\n",
    "        bert_optimizer.step()\n",
    "        head_optimizer.step()\n",
    "\n",
    "        print(f\"Loss: {loss:.2f}\")\n",
    "\n",
    "        preds = log_softmax.argmax(1)\n",
    "\n",
    "        preds_np = preds.detach().numpy()\n",
    "        labels_np = labels.detach().numpy()\n",
    "\n",
    "        batch_size = labels_np.shape[0]\n",
    "\n",
    "        # noinspection PyTypeChecker\n",
    "        accuracy = sum(labels_np == preds_np) / batch_size\n",
    "\n",
    "        print(f\"Predictions: {preds_np}, Labels: {labels_np}, Accuracy: {accuracy}\")\n",
    "        print(tweets)\n",
    "\n",
    "        summary.add_scalar('loss_train', loss, global_step=global_step)\n",
    "        summary.add_scalar('accuracy', accuracy, global_step=global_step)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Save the model and print out the parameters"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "odict_keys(['encoder.embeddings.word_embeddings.weight', 'encoder.embeddings.position_embeddings.weight', 'encoder.embeddings.token_type_embeddings.weight', 'encoder.embeddings.LayerNorm.weight', 'encoder.embeddings.LayerNorm.bias', 'encoder.encoder.layer.0.attention.self.query.weight', 'encoder.encoder.layer.0.attention.self.query.bias', 'encoder.encoder.layer.0.attention.self.key.weight', 'encoder.encoder.layer.0.attention.self.key.bias', 'encoder.encoder.layer.0.attention.self.value.weight', 'encoder.encoder.layer.0.attention.self.value.bias', 'encoder.encoder.layer.0.attention.output.dense.weight', 'encoder.encoder.layer.0.attention.output.dense.bias', 'encoder.encoder.layer.0.attention.output.LayerNorm.weight', 'encoder.encoder.layer.0.attention.output.LayerNorm.bias', 'encoder.encoder.layer.0.intermediate.dense.weight', 'encoder.encoder.layer.0.intermediate.dense.bias', 'encoder.encoder.layer.0.output.dense.weight', 'encoder.encoder.layer.0.output.dense.bias', 'encoder.encoder.layer.0.output.LayerNorm.weight', 'encoder.encoder.layer.0.output.LayerNorm.bias', 'encoder.encoder.layer.1.attention.self.query.weight', 'encoder.encoder.layer.1.attention.self.query.bias', 'encoder.encoder.layer.1.attention.self.key.weight', 'encoder.encoder.layer.1.attention.self.key.bias', 'encoder.encoder.layer.1.attention.self.value.weight', 'encoder.encoder.layer.1.attention.self.value.bias', 'encoder.encoder.layer.1.attention.output.dense.weight', 'encoder.encoder.layer.1.attention.output.dense.bias', 'encoder.encoder.layer.1.attention.output.LayerNorm.weight', 'encoder.encoder.layer.1.attention.output.LayerNorm.bias', 'encoder.encoder.layer.1.intermediate.dense.weight', 'encoder.encoder.layer.1.intermediate.dense.bias', 'encoder.encoder.layer.1.output.dense.weight', 'encoder.encoder.layer.1.output.dense.bias', 'encoder.encoder.layer.1.output.LayerNorm.weight', 'encoder.encoder.layer.1.output.LayerNorm.bias', 'encoder.encoder.layer.2.attention.self.query.weight', 'encoder.encoder.layer.2.attention.self.query.bias', 'encoder.encoder.layer.2.attention.self.key.weight', 'encoder.encoder.layer.2.attention.self.key.bias', 'encoder.encoder.layer.2.attention.self.value.weight', 'encoder.encoder.layer.2.attention.self.value.bias', 'encoder.encoder.layer.2.attention.output.dense.weight', 'encoder.encoder.layer.2.attention.output.dense.bias', 'encoder.encoder.layer.2.attention.output.LayerNorm.weight', 'encoder.encoder.layer.2.attention.output.LayerNorm.bias', 'encoder.encoder.layer.2.intermediate.dense.weight', 'encoder.encoder.layer.2.intermediate.dense.bias', 'encoder.encoder.layer.2.output.dense.weight', 'encoder.encoder.layer.2.output.dense.bias', 'encoder.encoder.layer.2.output.LayerNorm.weight', 'encoder.encoder.layer.2.output.LayerNorm.bias', 'encoder.encoder.layer.3.attention.self.query.weight', 'encoder.encoder.layer.3.attention.self.query.bias', 'encoder.encoder.layer.3.attention.self.key.weight', 'encoder.encoder.layer.3.attention.self.key.bias', 'encoder.encoder.layer.3.attention.self.value.weight', 'encoder.encoder.layer.3.attention.self.value.bias', 'encoder.encoder.layer.3.attention.output.dense.weight', 'encoder.encoder.layer.3.attention.output.dense.bias', 'encoder.encoder.layer.3.attention.output.LayerNorm.weight', 'encoder.encoder.layer.3.attention.output.LayerNorm.bias', 'encoder.encoder.layer.3.intermediate.dense.weight', 'encoder.encoder.layer.3.intermediate.dense.bias', 'encoder.encoder.layer.3.output.dense.weight', 'encoder.encoder.layer.3.output.dense.bias', 'encoder.encoder.layer.3.output.LayerNorm.weight', 'encoder.encoder.layer.3.output.LayerNorm.bias', 'encoder.encoder.layer.4.attention.self.query.weight', 'encoder.encoder.layer.4.attention.self.query.bias', 'encoder.encoder.layer.4.attention.self.key.weight', 'encoder.encoder.layer.4.attention.self.key.bias', 'encoder.encoder.layer.4.attention.self.value.weight', 'encoder.encoder.layer.4.attention.self.value.bias', 'encoder.encoder.layer.4.attention.output.dense.weight', 'encoder.encoder.layer.4.attention.output.dense.bias', 'encoder.encoder.layer.4.attention.output.LayerNorm.weight', 'encoder.encoder.layer.4.attention.output.LayerNorm.bias', 'encoder.encoder.layer.4.intermediate.dense.weight', 'encoder.encoder.layer.4.intermediate.dense.bias', 'encoder.encoder.layer.4.output.dense.weight', 'encoder.encoder.layer.4.output.dense.bias', 'encoder.encoder.layer.4.output.LayerNorm.weight', 'encoder.encoder.layer.4.output.LayerNorm.bias', 'encoder.encoder.layer.5.attention.self.query.weight', 'encoder.encoder.layer.5.attention.self.query.bias', 'encoder.encoder.layer.5.attention.self.key.weight', 'encoder.encoder.layer.5.attention.self.key.bias', 'encoder.encoder.layer.5.attention.self.value.weight', 'encoder.encoder.layer.5.attention.self.value.bias', 'encoder.encoder.layer.5.attention.output.dense.weight', 'encoder.encoder.layer.5.attention.output.dense.bias', 'encoder.encoder.layer.5.attention.output.LayerNorm.weight', 'encoder.encoder.layer.5.attention.output.LayerNorm.bias', 'encoder.encoder.layer.5.intermediate.dense.weight', 'encoder.encoder.layer.5.intermediate.dense.bias', 'encoder.encoder.layer.5.output.dense.weight', 'encoder.encoder.layer.5.output.dense.bias', 'encoder.encoder.layer.5.output.LayerNorm.weight', 'encoder.encoder.layer.5.output.LayerNorm.bias', 'encoder.encoder.layer.6.attention.self.query.weight', 'encoder.encoder.layer.6.attention.self.query.bias', 'encoder.encoder.layer.6.attention.self.key.weight', 'encoder.encoder.layer.6.attention.self.key.bias', 'encoder.encoder.layer.6.attention.self.value.weight', 'encoder.encoder.layer.6.attention.self.value.bias', 'encoder.encoder.layer.6.attention.output.dense.weight', 'encoder.encoder.layer.6.attention.output.dense.bias', 'encoder.encoder.layer.6.attention.output.LayerNorm.weight', 'encoder.encoder.layer.6.attention.output.LayerNorm.bias', 'encoder.encoder.layer.6.intermediate.dense.weight', 'encoder.encoder.layer.6.intermediate.dense.bias', 'encoder.encoder.layer.6.output.dense.weight', 'encoder.encoder.layer.6.output.dense.bias', 'encoder.encoder.layer.6.output.LayerNorm.weight', 'encoder.encoder.layer.6.output.LayerNorm.bias', 'encoder.encoder.layer.7.attention.self.query.weight', 'encoder.encoder.layer.7.attention.self.query.bias', 'encoder.encoder.layer.7.attention.self.key.weight', 'encoder.encoder.layer.7.attention.self.key.bias', 'encoder.encoder.layer.7.attention.self.value.weight', 'encoder.encoder.layer.7.attention.self.value.bias', 'encoder.encoder.layer.7.attention.output.dense.weight', 'encoder.encoder.layer.7.attention.output.dense.bias', 'encoder.encoder.layer.7.attention.output.LayerNorm.weight', 'encoder.encoder.layer.7.attention.output.LayerNorm.bias', 'encoder.encoder.layer.7.intermediate.dense.weight', 'encoder.encoder.layer.7.intermediate.dense.bias', 'encoder.encoder.layer.7.output.dense.weight', 'encoder.encoder.layer.7.output.dense.bias', 'encoder.encoder.layer.7.output.LayerNorm.weight', 'encoder.encoder.layer.7.output.LayerNorm.bias', 'encoder.encoder.layer.8.attention.self.query.weight', 'encoder.encoder.layer.8.attention.self.query.bias', 'encoder.encoder.layer.8.attention.self.key.weight', 'encoder.encoder.layer.8.attention.self.key.bias', 'encoder.encoder.layer.8.attention.self.value.weight', 'encoder.encoder.layer.8.attention.self.value.bias', 'encoder.encoder.layer.8.attention.output.dense.weight', 'encoder.encoder.layer.8.attention.output.dense.bias', 'encoder.encoder.layer.8.attention.output.LayerNorm.weight', 'encoder.encoder.layer.8.attention.output.LayerNorm.bias', 'encoder.encoder.layer.8.intermediate.dense.weight', 'encoder.encoder.layer.8.intermediate.dense.bias', 'encoder.encoder.layer.8.output.dense.weight', 'encoder.encoder.layer.8.output.dense.bias', 'encoder.encoder.layer.8.output.LayerNorm.weight', 'encoder.encoder.layer.8.output.LayerNorm.bias', 'encoder.encoder.layer.9.attention.self.query.weight', 'encoder.encoder.layer.9.attention.self.query.bias', 'encoder.encoder.layer.9.attention.self.key.weight', 'encoder.encoder.layer.9.attention.self.key.bias', 'encoder.encoder.layer.9.attention.self.value.weight', 'encoder.encoder.layer.9.attention.self.value.bias', 'encoder.encoder.layer.9.attention.output.dense.weight', 'encoder.encoder.layer.9.attention.output.dense.bias', 'encoder.encoder.layer.9.attention.output.LayerNorm.weight', 'encoder.encoder.layer.9.attention.output.LayerNorm.bias', 'encoder.encoder.layer.9.intermediate.dense.weight', 'encoder.encoder.layer.9.intermediate.dense.bias', 'encoder.encoder.layer.9.output.dense.weight', 'encoder.encoder.layer.9.output.dense.bias', 'encoder.encoder.layer.9.output.LayerNorm.weight', 'encoder.encoder.layer.9.output.LayerNorm.bias', 'encoder.encoder.layer.10.attention.self.query.weight', 'encoder.encoder.layer.10.attention.self.query.bias', 'encoder.encoder.layer.10.attention.self.key.weight', 'encoder.encoder.layer.10.attention.self.key.bias', 'encoder.encoder.layer.10.attention.self.value.weight', 'encoder.encoder.layer.10.attention.self.value.bias', 'encoder.encoder.layer.10.attention.output.dense.weight', 'encoder.encoder.layer.10.attention.output.dense.bias', 'encoder.encoder.layer.10.attention.output.LayerNorm.weight', 'encoder.encoder.layer.10.attention.output.LayerNorm.bias', 'encoder.encoder.layer.10.intermediate.dense.weight', 'encoder.encoder.layer.10.intermediate.dense.bias', 'encoder.encoder.layer.10.output.dense.weight', 'encoder.encoder.layer.10.output.dense.bias', 'encoder.encoder.layer.10.output.LayerNorm.weight', 'encoder.encoder.layer.10.output.LayerNorm.bias', 'encoder.encoder.layer.11.attention.self.query.weight', 'encoder.encoder.layer.11.attention.self.query.bias', 'encoder.encoder.layer.11.attention.self.key.weight', 'encoder.encoder.layer.11.attention.self.key.bias', 'encoder.encoder.layer.11.attention.self.value.weight', 'encoder.encoder.layer.11.attention.self.value.bias', 'encoder.encoder.layer.11.attention.output.dense.weight', 'encoder.encoder.layer.11.attention.output.dense.bias', 'encoder.encoder.layer.11.attention.output.LayerNorm.weight', 'encoder.encoder.layer.11.attention.output.LayerNorm.bias', 'encoder.encoder.layer.11.intermediate.dense.weight', 'encoder.encoder.layer.11.intermediate.dense.bias', 'encoder.encoder.layer.11.output.dense.weight', 'encoder.encoder.layer.11.output.dense.bias', 'encoder.encoder.layer.11.output.LayerNorm.weight', 'encoder.encoder.layer.11.output.LayerNorm.bias', 'encoder.pooler.dense.weight', 'encoder.pooler.dense.bias', 'classes.weight', 'classes.bias'])\n"
     ]
    }
   ],
   "source": [
    "# Let's save the model (we'll use it in the next video)\n",
    "torch.save(classifier.state_dict(), os.path.join(data_dir,'classifier_model.pt'))\n",
    "\n",
    "# Let's print out the state dict\n",
    "print(classifier.state_dict().keys())"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Look at the curves in TensorBoard\n",
    "\n",
    "`tensorboard --logdir=./`\n",
    "\n",
    "then go to http://localhost:6006/"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "You should see graphs that look like this\n",
    "![Tensorboard graphs](./tensorboard.png)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# In the next video, we'll look at these vectors in 3D space in TensorBoard and compare them with the vectors created in episode 1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(101, 768)\n"
     ]
    }
   ],
   "source": [
    "import numpy as np\n",
    "import pickle\n",
    "\n",
    "classifier.eval()  # needed when the model uses things like dropout\n",
    "\n",
    "\n",
    "all_vectors_np = []\n",
    "all_tweets = []\n",
    "for input_tuple in train_data_loader:\n",
    "\n",
    "        input_data, mask_data, labels, tweets = input_tuple\n",
    "        _log_softmax, vectors = classifier(input_data, mask_data)\n",
    "        \n",
    "        vectors_np = vectors.detach().numpy()\n",
    "        all_vectors_np.append(vectors_np)\n",
    "        all_tweets.extend(tweets)\n",
    "\n",
    "all_vecs = np.concatenate(all_vectors_np, axis=0)\n",
    "print(all_vecs.shape)\n",
    "\n",
    "\n",
    "custom_data = {'tweets':all_tweets, 'vecs':all_vecs}\n",
    "\n",
    "custom_vec_pickle_file = os.path.join(data_dir, 'custom_vecs.pkl')\n",
    "with open(custom_vec_pickle_file, 'wb') as f:\n",
    "        pickle.dump(custom_data, f)\n",
    "\n"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3.7.4 64-bit ('base': conda)",
   "language": "python",
   "name": "python37464bitbaseconda6a86cbe610af4db3860115ffdb24f1cc"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}