MLZoomcamp Capstone NLP IMDB Rating Predictor.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "2d3f7ab5",
   "metadata": {},
   "outputs": [],
   "source": [
    "# pip install datasets transformers[sentencepiece]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "0871c460",
   "metadata": {},
   "outputs": [],
   "source": [
    "# pip install tensorflow"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "3c97e22b",
   "metadata": {},
   "outputs": [],
   "source": [
    "from datasets import load_dataset"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "822222b0",
   "metadata": {},
   "source": [
    "The \\u0085 unicode character in some of the text files was causing the files to be split into multiple rows. So I need to remove this character from the text files."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 73,
   "id": "573d586e",
   "metadata": {},
   "outputs": [],
   "source": [
    "last_hidden_state = distilBERT([input_ids_layer, input_attention_layer])[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 95,
   "id": "319fb878",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<KerasTensor: shape=(None, 512, 768) dtype=float32 (created by layer 'tf_distil_bert_model_1')>"
      ]
     },
     "execution_count": 95,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "last_hidden_state"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 115,
   "id": "eb3084ae",
   "metadata": {},
   "outputs": [],
   "source": [
    "input_arr = tf.keras.layers.GlobalAvgPool1D()(last_hidden_state)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 116,
   "id": "febcbc84",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Assets written to: my_model/assets\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Assets written to: my_model/assets\n"
     ]
    }
   ],
   "source": [
    "outputs = model(input_arr)\n",
    "model.save(\"my_model\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 117,
   "id": "ad97b44b",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Model: \"custom_model_4\"\n",
      "_________________________________________________________________\n",
      " Layer (type)                Output Shape              Param #   \n",
      "=================================================================\n",
      " dense_25 (Dense)            multiple                  98432     \n",
      "                                                                 \n",
      " dense_26 (Dense)            multiple                  16512     \n",
      "                                                                 \n",
      " dense_27 (Dense)            multiple                  8256      \n",
      "                                                                 \n",
      " dense_28 (Dense)            multiple                  650       \n",
      "                                                                 \n",
      " dense_29 (Dense)            multiple                  11        \n",
      "                                                                 \n",
      "=================================================================\n",
      "Total params: 123,861\n",
      "Trainable params: 123,861\n",
      "Non-trainable params: 0\n",
      "_________________________________________________________________\n"
     ]
    }
   ],
   "source": [
    "model.summary()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 121,
   "id": "2c3f09cb",
   "metadata": {},
   "outputs": [],
   "source": [
    "model.layers[0]._name = 'test'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 77,
   "id": "b273f390",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:No training configuration found in save file, so the model was *not* compiled. Compile it manually.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:No training configuration found in save file, so the model was *not* compiled. Compile it manually.\n"
     ]
    }
   ],
   "source": [
    "loaded_1 = keras.models.load_model(\n",
    "    \"my_model\", custom_objects={\"CustomModel\": CustomModel}\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 78,
   "id": "36a33770",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Model: \"custom_model_1\"\n",
      "_________________________________________________________________\n",
      " Layer (type)                Output Shape              Param #   \n",
      "=================================================================\n",
      " dense_15 (Dense)            multiple                  98432     \n",
      "                                                                 \n",
      " dense_16 (Dense)            multiple                  16512     \n",
      "                                                                 \n",
      " dense_17 (Dense)            multiple                  8256      \n",
      "                                                                 \n",
      " dense_18 (Dense)            multiple                  650       \n",
      "                                                                 \n",
      " dense_19 (Dense)            multiple                  11        \n",
      "                                                                 \n",
      "=================================================================\n",
      "Total params: 123,861\n",
      "Trainable params: 123,861\n",
      "Non-trainable params: 0\n",
      "_________________________________________________________________\n"
     ]
    }
   ],
   "source": [
    "loaded_1.summary()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 92,
   "id": "4973939c",
   "metadata": {},
   "outputs": [],
   "source": [
    "loaded_1.load_weights('models/final_ratings_model_mse_4.054.h5', by_name=True, skip_mismatch=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "id": "a1f3e5fc",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Some layers from the model checkpoint at distilbert-base-uncased were not used when initializing TFDistilBertModel: ['activation_13', 'vocab_layer_norm', 'vocab_transform', 'vocab_projector']\n",
      "- This IS expected if you are initializing TFDistilBertModel from the checkpoint of a model trained on another task or with another architecture (e.g. initializing a BertForSequenceClassification model from a BertForPreTraining model).\n",
      "- This IS NOT expected if you are initializing TFDistilBertModel from the checkpoint of a model that you expect to be exactly identical (initializing a BertForSequenceClassification model from a BertForSequenceClassification model).\n",
      "All the layers of TFDistilBertModel were initialized from the model checkpoint at distilbert-base-uncased.\n",
      "If your task is similar to the task the model of the checkpoint was trained on, you can already use TFDistilBertModel for predictions without further training.\n"
     ]
    }
   ],
   "source": [
    "my_model = TFDistilBertModel.from_pretrained('distilbert-base-uncased')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "id": "51e21d5c",
   "metadata": {},
   "outputs": [],
   "source": [
    "TFDistilBertModel.save_pretrained(my_model, 'saved_model', save_format = 'tf')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 57,
   "id": "1850be18",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "All model checkpoint layers were used when initializing TFDistilBertModel.\n",
      "\n",
      "All the layers of TFDistilBertModel were initialized from the model checkpoint at saved_model/.\n",
      "If your task is similar to the task the model of the checkpoint was trained on, you can already use TFDistilBertModel for predictions without further training.\n"
     ]
    }
   ],
   "source": [
    "new_model = TFDistilBertModel.from_pretrained('saved_model/')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 60,
   "id": "99fb9a32",
   "metadata": {},
   "outputs": [],
   "source": [
    "    input_ids_layer = tf.keras.layers.Input(shape=(params['max_length'],), \n",
    "                                            name='input_ids', \n",
    "                                            dtype='int32')\n",
    "    input_attention_layer = tf.keras.layers.Input(shape=(params['max_length'],), \n",
    "                                                  name='attention_mask', \n",
    "                                                  dtype='int32')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 61,
   "id": "fc23c50a",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<KerasTensor: shape=(None, 512, 768) dtype=float32 (created by layer 'tf_distil_bert_model_3')>"
      ]
     },
     "execution_count": 61,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "new_model([input_ids_layer, input_attention_layer])[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 87,
   "id": "332b77b2",
   "metadata": {},
   "outputs": [],
   "source": [
    "text = \"\"\"\n",
    "It was amazing!\n",
    "\"\"\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 88,
   "id": "d54a365e",
   "metadata": {},
   "outputs": [],
   "source": [
    "tensor = tokenizer(text, truncation=True, padding='max_length', max_length=params['max_length'], return_tensors='tf')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 94,
   "id": "98367f79",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[2.020507]], dtype=float32)"
      ]
     },
     "execution_count": 94,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "loaded_1.predict(tf.keras.layers.GlobalAvgPool1D()(distilBERT(tensor.data)[0]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "ee4a99c5",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.8"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}