notebook_12_data_modeling_with_LSTM.ipynb

{"cells":[{"cell_type":"markdown","metadata":{"id":"mzN7ShGenbbH"},"source":["## Data Modeling and Evaluation Steps (model 3)\n","\n","At this moment, we are going to create and implement the *Long Short Term Memory* (LSTM) model.\n"]},{"cell_type":"code","execution_count":2,"metadata":{"colab":{"base_uri":"https://localhost:8080/"},"executionInfo":{"elapsed":92587,"status":"ok","timestamp":1679695873949,"user":{"displayName":"Rafaela Queiroz F. Cordeiro","userId":"18015024781067059704"},"user_tz":420},"id":"2BxJFjTbF7Df","outputId":"0c1a15c0-13e8-42fc-806d-14d3afc74ef4"},"outputs":[{"name":"stdout","output_type":"stream","text":["Mounted at /content/drive\n"]}],"source":["from google.colab import drive\n","drive.mount('/content/drive')"]},{"cell_type":"markdown","metadata":{"id":"gNYLy2VDHFPY"},"source":["#### Step 1: Retake the *X* and *y* variables from our trainning set"]},{"cell_type":"code","execution_count":1,"metadata":{"executionInfo":{"elapsed":641,"status":"ok","timestamp":1679695874575,"user":{"displayName":"Rafaela Queiroz F. Cordeiro","userId":"18015024781067059704"},"user_tz":420},"id":"rI8MucyQG5qV"},"outputs":[],"source":["# Import basic libraries\n","import pandas as pd\n","import numpy as np"]},{"cell_type":"code","execution_count":2,"metadata":{"colab":{"base_uri":"https://localhost:8080/","height":424},"executionInfo":{"elapsed":1708,"status":"ok","timestamp":1679695876271,"user":{"displayName":"Rafaela Queiroz F. Cordeiro","userId":"18015024781067059704"},"user_tz":420},"id":"FafwwDBQFv73","outputId":"25101ffe-b9a8-4898-baa0-951629489279"},"outputs":[{"data":{"text/html":["<div>\n","<style scoped>\n","    .dataframe tbody tr th:only-of-type {\n","        vertical-align: middle;\n","    }\n","\n","    .dataframe tbody tr th {\n","        vertical-align: top;\n","    }\n","\n","    .dataframe thead th {\n","        text-align: right;\n","    }\n","</style>\n","<table border=\"1\" class=\"dataframe\">\n","  <thead>\n","    <tr style=\"text-align: right;\">\n","      <th></th>\n","      <th>drugName</th>\n","      <th>condition</th>\n","      <th>rating</th>\n","      <th>date</th>\n","      <th>usefulCount</th>\n","      <th>year</th>\n","      <th>review_word_lemm</th>\n","      <th>polarity</th>\n","      <th>rating_classification</th>\n","    </tr>\n","  </thead>\n","  <tbody>\n","    <tr>\n","      <th>0</th>\n","      <td>Valsartan</td>\n","      <td>Left Ventricular Dysfunction</td>\n","      <td>9.0</td>\n","      <td>2012-05-20</td>\n","      <td>27</td>\n","      <td>2012</td>\n","      <td>['no', 'side', 'effect', 'take', 'combination'...</td>\n","      <td>0.000000</td>\n","      <td>2</td>\n","    </tr>\n","    <tr>\n","      <th>1</th>\n","      <td>Guanfacine</td>\n","      <td>ADHD</td>\n","      <td>8.0</td>\n","      <td>2010-04-27</td>\n","      <td>192</td>\n","      <td>2010</td>\n","      <td>['son', 'halfway', 'fourth', 'week', 'intuniv'...</td>\n","      <td>0.188021</td>\n","      <td>2</td>\n","    </tr>\n","    <tr>\n","      <th>2</th>\n","      <td>Lybrel</td>\n","      <td>Birth Control</td>\n","      <td>5.0</td>\n","      <td>2009-12-14</td>\n","      <td>17</td>\n","      <td>2009</td>\n","      <td>['used', 'take', 'another', 'oral', 'contracep...</td>\n","      <td>0.113636</td>\n","      <td>1</td>\n","    </tr>\n","    <tr>\n","      <th>3</th>\n","      <td>Ortho Evra</td>\n","      <td>Birth Control</td>\n","      <td>8.0</td>\n","      <td>2015-11-03</td>\n","      <td>10</td>\n","      <td>2015</td>\n","      <td>['first', 'time', 'using', 'form', 'birth', 'c...</td>\n","      <td>0.262500</td>\n","      <td>2</td>\n","    </tr>\n","    <tr>\n","      <th>4</th>\n","      <td>Buprenorphine / naloxone</td>\n","      <td>Opiate Dependence</td>\n","      <td>9.0</td>\n","      <td>2016-11-27</td>\n","      <td>37</td>\n","      <td>2016</td>\n","      <td>['suboxone', 'completely', 'turned', 'life', '...</td>\n","      <td>0.163333</td>\n","      <td>2</td>\n","    </tr>\n","    <tr>\n","      <th>...</th>\n","      <td>...</td>\n","      <td>...</td>\n","      <td>...</td>\n","      <td>...</td>\n","      <td>...</td>\n","      <td>...</td>\n","      <td>...</td>\n","      <td>...</td>\n","      <td>...</td>\n","    </tr>\n","    <tr>\n","      <th>112324</th>\n","      <td>Carbamazepine</td>\n","      <td>Trigeminal Neuralgia</td>\n","      <td>1.0</td>\n","      <td>2016-01-31</td>\n","      <td>10</td>\n","      <td>2016</td>\n","      <td>['mg', 'seems', 'work', 'every', 'nd', 'day', ...</td>\n","      <td>0.000000</td>\n","      <td>0</td>\n","    </tr>\n","    <tr>\n","      <th>112325</th>\n","      <td>Tekturna</td>\n","      <td>High Blood Pressure</td>\n","      <td>7.0</td>\n","      <td>2010-02-07</td>\n","      <td>18</td>\n","      <td>2010</td>\n","      <td>['tekturna', 'day', 'effect', 'immediate', 'al...</td>\n","      <td>-0.087500</td>\n","      <td>2</td>\n","    </tr>\n","    <tr>\n","      <th>112326</th>\n","      <td>Campral</td>\n","      <td>Alcohol Dependence</td>\n","      <td>10.0</td>\n","      <td>2015-05-31</td>\n","      <td>125</td>\n","      <td>2015</td>\n","      <td>['wrote', 'first', 'report', 'midoctober', 'no...</td>\n","      <td>0.261905</td>\n","      <td>2</td>\n","    </tr>\n","    <tr>\n","      <th>112327</th>\n","      <td>Thyroid desiccated</td>\n","      <td>Underactive Thyroid</td>\n","      <td>10.0</td>\n","      <td>2015-09-19</td>\n","      <td>79</td>\n","      <td>2015</td>\n","      <td>['ive', 'thyroid', 'medication', 'year', 'spen...</td>\n","      <td>0.201313</td>\n","      <td>2</td>\n","    </tr>\n","    <tr>\n","      <th>112328</th>\n","      <td>Lubiprostone</td>\n","      <td>Constipation, Chronic</td>\n","      <td>9.0</td>\n","      <td>2014-12-13</td>\n","      <td>116</td>\n","      <td>2014</td>\n","      <td>['ive', 'chronic', 'constipation', 'adult', 'l...</td>\n","      <td>0.104167</td>\n","      <td>2</td>\n","    </tr>\n","  </tbody>\n","</table>\n","<p>112329 rows × 9 columns</p>\n","</div>"],"text/plain":["                        drugName                     condition  rating  \\\n","0                      Valsartan  Left Ventricular Dysfunction     9.0   \n","1                     Guanfacine                          ADHD     8.0   \n","2                         Lybrel                 Birth Control     5.0   \n","3                     Ortho Evra                 Birth Control     8.0   \n","4       Buprenorphine / naloxone             Opiate Dependence     9.0   \n","...                          ...                           ...     ...   \n","112324             Carbamazepine          Trigeminal Neuralgia     1.0   \n","112325                  Tekturna           High Blood Pressure     7.0   \n","112326                   Campral            Alcohol Dependence    10.0   \n","112327        Thyroid desiccated           Underactive Thyroid    10.0   \n","112328              Lubiprostone         Constipation, Chronic     9.0   \n","\n","              date  usefulCount  year  \\\n","0       2012-05-20           27  2012   \n","1       2010-04-27          192  2010   \n","2       2009-12-14           17  2009   \n","3       2015-11-03           10  2015   \n","4       2016-11-27           37  2016   \n","...            ...          ...   ...   \n","112324  2016-01-31           10  2016   \n","112325  2010-02-07           18  2010   \n","112326  2015-05-31          125  2015   \n","112327  2015-09-19           79  2015   \n","112328  2014-12-13          116  2014   \n","\n","                                         review_word_lemm  polarity  \\\n","0       ['no', 'side', 'effect', 'take', 'combination'...  0.000000   \n","1       ['son', 'halfway', 'fourth', 'week', 'intuniv'...  0.188021   \n","2       ['used', 'take', 'another', 'oral', 'contracep...  0.113636   \n","3       ['first', 'time', 'using', 'form', 'birth', 'c...  0.262500   \n","4       ['suboxone', 'completely', 'turned', 'life', '...  0.163333   \n","...                                                   ...       ...   \n","112324  ['mg', 'seems', 'work', 'every', 'nd', 'day', ...  0.000000   \n","112325  ['tekturna', 'day', 'effect', 'immediate', 'al... -0.087500   \n","112326  ['wrote', 'first', 'report', 'midoctober', 'no...  0.261905   \n","112327  ['ive', 'thyroid', 'medication', 'year', 'spen...  0.201313   \n","112328  ['ive', 'chronic', 'constipation', 'adult', 'l...  0.104167   \n","\n","        rating_classification  \n","0                           2  \n","1                           2  \n","2                           1  \n","3                           2  \n","4                           2  \n","...                       ...  \n","112324                      0  \n","112325                      2  \n","112326                      2  \n","112327                      2  \n","112328                      2  \n","\n","[112329 rows x 9 columns]"]},"execution_count":2,"metadata":{},"output_type":"execute_result"}],"source":["# Load the trainning dataset\n","# train_medication_reviews = pd.read_csv('/content/drive/Othercomputers/My MacBook Pro/Sentiment-Analysis-of-Medication-Reviews-Project/medication_reviews_dataset_to_train.csv', sep=',')\n","train_medication_reviews = pd.read_csv('/Users/rafaelaqueiroz/Sentiment-Analysis-of-Medication-Reviews-Project/medication_reviews_dataset_to_train.csv', sep=',')\n","train_medication_reviews"]},{"cell_type":"code","execution_count":3,"metadata":{"colab":{"base_uri":"https://localhost:8080/"},"executionInfo":{"elapsed":113,"status":"ok","timestamp":1679695876274,"user":{"displayName":"Rafaela Queiroz F. Cordeiro","userId":"18015024781067059704"},"user_tz":420},"id":"23Sax0ZrHCyY","outputId":"df1581ec-7d56-4a3c-8815-b1d3508a5f0e"},"outputs":[{"data":{"text/plain":["0         ['no', 'side', 'effect', 'take', 'combination'...\n","1         ['son', 'halfway', 'fourth', 'week', 'intuniv'...\n","2         ['used', 'take', 'another', 'oral', 'contracep...\n","3         ['first', 'time', 'using', 'form', 'birth', 'c...\n","4         ['suboxone', 'completely', 'turned', 'life', '...\n","                                ...                        \n","112324    ['mg', 'seems', 'work', 'every', 'nd', 'day', ...\n","112325    ['tekturna', 'day', 'effect', 'immediate', 'al...\n","112326    ['wrote', 'first', 'report', 'midoctober', 'no...\n","112327    ['ive', 'thyroid', 'medication', 'year', 'spen...\n","112328    ['ive', 'chronic', 'constipation', 'adult', 'l...\n","Name: review_word_lemm, Length: 112329, dtype: object"]},"execution_count":3,"metadata":{},"output_type":"execute_result"}],"source":["# As we already know from our previous notebook (notebook 3), our independent variable (X) is going to be the \"review_word_lemm\" variable\n","X_train = train_medication_reviews.review_word_lemm\n","X_train"]},{"cell_type":"code","execution_count":5,"metadata":{"colab":{"base_uri":"https://localhost:8080/"},"executionInfo":{"elapsed":100,"status":"ok","timestamp":1679695876276,"user":{"displayName":"Rafaela Queiroz F. Cordeiro","userId":"18015024781067059704"},"user_tz":420},"id":"6J1QsRyCHCns","outputId":"dedd8600-3454-46c5-e5e8-2cd1af49fb44"},"outputs":[{"data":{"text/plain":["(112329,)"]},"execution_count":5,"metadata":{},"output_type":"execute_result"}],"source":["X_train.shape"]},{"cell_type":"code","execution_count":6,"metadata":{"colab":{"base_uri":"https://localhost:8080/"},"executionInfo":{"elapsed":93,"status":"ok","timestamp":1679695876278,"user":{"displayName":"Rafaela Queiroz F. Cordeiro","userId":"18015024781067059704"},"user_tz":420},"id":"rk0XG7kVL8ZC","outputId":"215b5b67-7d37-4598-fdd2-ac4a3ecbbafa"},"outputs":[{"data":{"text/plain":["pandas.core.series.Series"]},"execution_count":6,"metadata":{},"output_type":"execute_result"}],"source":["type(X_train)"]},{"cell_type":"code","execution_count":7,"metadata":{"colab":{"base_uri":"https://localhost:8080/","height":107},"executionInfo":{"elapsed":90,"status":"ok","timestamp":1679695876281,"user":{"displayName":"Rafaela Queiroz F. Cordeiro","userId":"18015024781067059704"},"user_tz":420},"id":"LjFDBjbPfrbq","outputId":"4d595400-ed36-4cae-9a98-32f51db79163"},"outputs":[{"data":{"text/plain":["\"['no', 'side', 'effect', 'take', 'combination', 'bystolic', 'mg', 'fish', 'oil']\""]},"execution_count":7,"metadata":{},"output_type":"execute_result"}],"source":["X_train[0]"]},{"cell_type":"code","execution_count":4,"metadata":{"colab":{"base_uri":"https://localhost:8080/"},"executionInfo":{"elapsed":88,"status":"ok","timestamp":1679695876282,"user":{"displayName":"Rafaela Queiroz F. Cordeiro","userId":"18015024781067059704"},"user_tz":420},"id":"nRYu0qGSMQAh","outputId":"1ace1d13-8fe4-470d-8368-cd1d4a031c52"},"outputs":[{"data":{"text/plain":["0         2\n","1         2\n","2         1\n","3         2\n","4         2\n","         ..\n","112324    0\n","112325    2\n","112326    2\n","112327    2\n","112328    2\n","Name: rating_classification, Length: 112329, dtype: int64"]},"execution_count":4,"metadata":{},"output_type":"execute_result"}],"source":["# As we know, our target or dependent variable (y) is going to be the 'rating_classification' variable\n","y_train = train_medication_reviews.rating_classification\n","y_train"]},{"cell_type":"code","execution_count":9,"metadata":{"colab":{"base_uri":"https://localhost:8080/"},"executionInfo":{"elapsed":84,"status":"ok","timestamp":1679695876282,"user":{"displayName":"Rafaela Queiroz F. Cordeiro","userId":"18015024781067059704"},"user_tz":420},"id":"xWwZH4W4MW6A","outputId":"8f100bb2-1c2a-4fd1-968c-63c6e92b7745"},"outputs":[{"data":{"text/plain":["(112329,)"]},"execution_count":9,"metadata":{},"output_type":"execute_result"}],"source":["y_train.shape"]},{"cell_type":"code","execution_count":10,"metadata":{"colab":{"base_uri":"https://localhost:8080/"},"executionInfo":{"elapsed":81,"status":"ok","timestamp":1679695876283,"user":{"displayName":"Rafaela Queiroz F. Cordeiro","userId":"18015024781067059704"},"user_tz":420},"id":"MiJkJUnNMaEl","outputId":"ad7f2642-0d1e-4440-f891-c191c175e3c3"},"outputs":[{"data":{"text/plain":["pandas.core.series.Series"]},"execution_count":10,"metadata":{},"output_type":"execute_result"}],"source":["type(y_train)"]},{"cell_type":"markdown","metadata":{"id":"x80AucaFORO-"},"source":["#### Step 2: Create the *X* and *y* variables from our testing set\n","\n","Note: As this is the testing set, we are not going to apply any cleaning or processint to it. However, as we have done label encoding to the *rating* column, we would need to label encoding this column at this set as well since the model cannot predict the sentiment of the reviews from rating 1 to 10 as we are representing those numbers differently, such as: 0 - negative reviews, 1 - \"neutral\" reviews, and 2 - positive reviews."]},{"cell_type":"code","execution_count":5,"metadata":{"colab":{"base_uri":"https://localhost:8080/","height":206},"executionInfo":{"elapsed":1892,"status":"ok","timestamp":1679695878098,"user":{"displayName":"Rafaela Queiroz F. Cordeiro","userId":"18015024781067059704"},"user_tz":420},"id":"_oYUAe0gFzUD","outputId":"bce94d86-608f-4462-906c-68fda1e08609"},"outputs":[{"data":{"text/html":["<div>\n","<style scoped>\n","    .dataframe tbody tr th:only-of-type {\n","        vertical-align: middle;\n","    }\n","\n","    .dataframe tbody tr th {\n","        vertical-align: top;\n","    }\n","\n","    .dataframe thead th {\n","        text-align: right;\n","    }\n","</style>\n","<table border=\"1\" class=\"dataframe\">\n","  <thead>\n","    <tr style=\"text-align: right;\">\n","      <th></th>\n","      <th>Unnamed: 0</th>\n","      <th>drugName</th>\n","      <th>condition</th>\n","      <th>review</th>\n","      <th>rating</th>\n","      <th>date</th>\n","      <th>usefulCount</th>\n","    </tr>\n","  </thead>\n","  <tbody>\n","    <tr>\n","      <th>0</th>\n","      <td>163740</td>\n","      <td>Mirtazapine</td>\n","      <td>Depression</td>\n","      <td>\"I&amp;#039;ve tried a few antidepressants over th...</td>\n","      <td>10.0</td>\n","      <td>February 28, 2012</td>\n","      <td>22</td>\n","    </tr>\n","    <tr>\n","      <th>1</th>\n","      <td>206473</td>\n","      <td>Mesalamine</td>\n","      <td>Crohn's Disease, Maintenance</td>\n","      <td>\"My son has Crohn&amp;#039;s disease and has done ...</td>\n","      <td>8.0</td>\n","      <td>May 17, 2009</td>\n","      <td>17</td>\n","    </tr>\n","    <tr>\n","      <th>2</th>\n","      <td>159672</td>\n","      <td>Bactrim</td>\n","      <td>Urinary Tract Infection</td>\n","      <td>\"Quick reduction of symptoms\"</td>\n","      <td>9.0</td>\n","      <td>September 29, 2017</td>\n","      <td>3</td>\n","    </tr>\n","    <tr>\n","      <th>3</th>\n","      <td>39293</td>\n","      <td>Contrave</td>\n","      <td>Weight Loss</td>\n","      <td>\"Contrave combines drugs that were used for al...</td>\n","      <td>9.0</td>\n","      <td>March 5, 2017</td>\n","      <td>35</td>\n","    </tr>\n","    <tr>\n","      <th>4</th>\n","      <td>97768</td>\n","      <td>Cyclafem 1 / 35</td>\n","      <td>Birth Control</td>\n","      <td>\"I have been on this birth control for one cyc...</td>\n","      <td>9.0</td>\n","      <td>October 22, 2015</td>\n","      <td>4</td>\n","    </tr>\n","  </tbody>\n","</table>\n","</div>"],"text/plain":["   Unnamed: 0         drugName                     condition  \\\n","0      163740      Mirtazapine                    Depression   \n","1      206473       Mesalamine  Crohn's Disease, Maintenance   \n","2      159672          Bactrim       Urinary Tract Infection   \n","3       39293         Contrave                   Weight Loss   \n","4       97768  Cyclafem 1 / 35                 Birth Control   \n","\n","                                              review  rating  \\\n","0  \"I&#039;ve tried a few antidepressants over th...    10.0   \n","1  \"My son has Crohn&#039;s disease and has done ...     8.0   \n","2                      \"Quick reduction of symptoms\"     9.0   \n","3  \"Contrave combines drugs that were used for al...     9.0   \n","4  \"I have been on this birth control for one cyc...     9.0   \n","\n","                 date  usefulCount  \n","0   February 28, 2012           22  \n","1        May 17, 2009           17  \n","2  September 29, 2017            3  \n","3       March 5, 2017           35  \n","4    October 22, 2015            4  "]},"execution_count":5,"metadata":{},"output_type":"execute_result"}],"source":["# Load the test dataset\n","# test_drug_reviews_df = pd.read_csv('/content/drive/MyDrive/Data-Science-Other-Materials/Data-Scientist-Bootcamp/Sentiment-Analysis-of-Drug-Reviews/drugsComTest_raw.tsv', delimiter='\\t')\n","test_drug_reviews_df = pd.read_csv('/Users/rafaelaqueiroz/Sentiment-Analysis-of-Medication-Reviews-Project/drugsComTest_raw.tsv', delimiter='\\t')\n","test_drug_reviews_df.head()"]},{"cell_type":"code","execution_count":12,"metadata":{"colab":{"base_uri":"https://localhost:8080/"},"executionInfo":{"elapsed":77,"status":"ok","timestamp":1679695878101,"user":{"displayName":"Rafaela Queiroz F. Cordeiro","userId":"18015024781067059704"},"user_tz":420},"id":"WwWkUdQARgqC","outputId":"7556474a-1d3d-4481-e15c-481a39e5379b"},"outputs":[{"data":{"text/plain":["(53766, 7)"]},"execution_count":12,"metadata":{},"output_type":"execute_result"}],"source":["test_drug_reviews_df.shape"]},{"cell_type":"code","execution_count":13,"metadata":{"colab":{"base_uri":"https://localhost:8080/"},"executionInfo":{"elapsed":73,"status":"ok","timestamp":1679695878103,"user":{"displayName":"Rafaela Queiroz F. Cordeiro","userId":"18015024781067059704"},"user_tz":420},"id":"LJ15F9HXU6RY","outputId":"2095e226-795a-4ca0-d1a6-d26667a35fdf"},"outputs":[{"data":{"text/plain":["pandas.core.frame.DataFrame"]},"execution_count":13,"metadata":{},"output_type":"execute_result"}],"source":["type(test_drug_reviews_df)"]},{"cell_type":"code","execution_count":6,"metadata":{"colab":{"base_uri":"https://localhost:8080/","height":206},"executionInfo":{"elapsed":1670,"status":"ok","timestamp":1679695879708,"user":{"displayName":"Rafaela Queiroz F. Cordeiro","userId":"18015024781067059704"},"user_tz":420},"id":"g9OP8It4OJI8","outputId":"4a244d43-ea2e-4001-bca3-61521e67cea8"},"outputs":[{"data":{"text/html":["<div>\n","<style scoped>\n","    .dataframe tbody tr th:only-of-type {\n","        vertical-align: middle;\n","    }\n","\n","    .dataframe tbody tr th {\n","        vertical-align: top;\n","    }\n","\n","    .dataframe thead th {\n","        text-align: right;\n","    }\n","</style>\n","<table border=\"1\" class=\"dataframe\">\n","  <thead>\n","    <tr style=\"text-align: right;\">\n","      <th></th>\n","      <th>Unnamed: 0</th>\n","      <th>drugName</th>\n","      <th>condition</th>\n","      <th>review</th>\n","      <th>rating</th>\n","      <th>date</th>\n","      <th>usefulCount</th>\n","      <th>rating_classification</th>\n","    </tr>\n","  </thead>\n","  <tbody>\n","    <tr>\n","      <th>0</th>\n","      <td>163740</td>\n","      <td>Mirtazapine</td>\n","      <td>Depression</td>\n","      <td>\"I&amp;#039;ve tried a few antidepressants over th...</td>\n","      <td>10.0</td>\n","      <td>February 28, 2012</td>\n","      <td>22</td>\n","      <td>2</td>\n","    </tr>\n","    <tr>\n","      <th>1</th>\n","      <td>206473</td>\n","      <td>Mesalamine</td>\n","      <td>Crohn's Disease, Maintenance</td>\n","      <td>\"My son has Crohn&amp;#039;s disease and has done ...</td>\n","      <td>8.0</td>\n","      <td>May 17, 2009</td>\n","      <td>17</td>\n","      <td>2</td>\n","    </tr>\n","    <tr>\n","      <th>2</th>\n","      <td>159672</td>\n","      <td>Bactrim</td>\n","      <td>Urinary Tract Infection</td>\n","      <td>\"Quick reduction of symptoms\"</td>\n","      <td>9.0</td>\n","      <td>September 29, 2017</td>\n","      <td>3</td>\n","      <td>2</td>\n","    </tr>\n","    <tr>\n","      <th>3</th>\n","      <td>39293</td>\n","      <td>Contrave</td>\n","      <td>Weight Loss</td>\n","      <td>\"Contrave combines drugs that were used for al...</td>\n","      <td>9.0</td>\n","      <td>March 5, 2017</td>\n","      <td>35</td>\n","      <td>2</td>\n","    </tr>\n","    <tr>\n","      <th>4</th>\n","      <td>97768</td>\n","      <td>Cyclafem 1 / 35</td>\n","      <td>Birth Control</td>\n","      <td>\"I have been on this birth control for one cyc...</td>\n","      <td>9.0</td>\n","      <td>October 22, 2015</td>\n","      <td>4</td>\n","      <td>2</td>\n","    </tr>\n","  </tbody>\n","</table>\n","</div>"],"text/plain":["   Unnamed: 0         drugName                     condition  \\\n","0      163740      Mirtazapine                    Depression   \n","1      206473       Mesalamine  Crohn's Disease, Maintenance   \n","2      159672          Bactrim       Urinary Tract Infection   \n","3       39293         Contrave                   Weight Loss   \n","4       97768  Cyclafem 1 / 35                 Birth Control   \n","\n","                                              review  rating  \\\n","0  \"I&#039;ve tried a few antidepressants over th...    10.0   \n","1  \"My son has Crohn&#039;s disease and has done ...     8.0   \n","2                      \"Quick reduction of symptoms\"     9.0   \n","3  \"Contrave combines drugs that were used for al...     9.0   \n","4  \"I have been on this birth control for one cyc...     9.0   \n","\n","                 date  usefulCount  rating_classification  \n","0   February 28, 2012           22                      2  \n","1        May 17, 2009           17                      2  \n","2  September 29, 2017            3                      2  \n","3       March 5, 2017           35                      2  \n","4    October 22, 2015            4                      2  "]},"execution_count":6,"metadata":{},"output_type":"execute_result"}],"source":["# Import library to label encode the rating column\n","from sklearn.preprocessing import LabelEncoder\n","\n","# Define the bin edges and labels (0 = 'negative', 1 = 'neutral', 2 = 'positive')\n","bin_edges = [0, 4, 6, 10]  # Ratings 1-4 are negative, 5-6 are neutral, 7-10 are positive\n","bin_labels = ['negative', 'neutral', 'positive'] # The rating_classification column should now only have 0, 1, or 2 values\n","\n","# Use cut to bin the \"rating\" column and create the new column called \"rating_classification\"\n","test_drug_reviews_df['rating_classification'] = pd.cut(test_drug_reviews_df['rating'], bins=bin_edges, labels=bin_labels)\n","\n","# Instantiate LabelEncoder and fit_transform the new column\n","le = LabelEncoder()\n","test_drug_reviews_df['rating_classification'] = le.fit_transform(test_drug_reviews_df['rating_classification'])\n","test_drug_reviews_df.head(5)"]},{"cell_type":"code","execution_count":15,"metadata":{"colab":{"base_uri":"https://localhost:8080/"},"executionInfo":{"elapsed":329,"status":"ok","timestamp":1679695879796,"user":{"displayName":"Rafaela Queiroz F. Cordeiro","userId":"18015024781067059704"},"user_tz":420},"id":"3BFPXjdtWd-p","outputId":"98f0d292-890e-4fd2-caa3-a668256af4ff"},"outputs":[{"data":{"text/plain":["pandas.core.frame.DataFrame"]},"execution_count":15,"metadata":{},"output_type":"execute_result"}],"source":["type(test_drug_reviews_df)"]},{"cell_type":"code","execution_count":16,"metadata":{"colab":{"base_uri":"https://localhost:8080/"},"executionInfo":{"elapsed":308,"status":"ok","timestamp":1679695879797,"user":{"displayName":"Rafaela Queiroz F. Cordeiro","userId":"18015024781067059704"},"user_tz":420},"id":"ZLYxDinWlj0Y","outputId":"1f7c796d-a658-45f0-812b-74ec24a0c47d"},"outputs":[{"data":{"text/plain":["(53766, 8)"]},"execution_count":16,"metadata":{},"output_type":"execute_result"}],"source":["test_drug_reviews_df.shape"]},{"cell_type":"code","execution_count":7,"metadata":{"colab":{"base_uri":"https://localhost:8080/"},"executionInfo":{"elapsed":303,"status":"ok","timestamp":1679695879798,"user":{"displayName":"Rafaela Queiroz F. Cordeiro","userId":"18015024781067059704"},"user_tz":420},"id":"abZlyIi4lnlF","outputId":"aa93a30c-8ea0-420b-c40a-41b4f0feda31"},"outputs":[{"data":{"text/plain":["0        \"I&#039;ve tried a few antidepressants over th...\n","1        \"My son has Crohn&#039;s disease and has done ...\n","2                            \"Quick reduction of symptoms\"\n","3        \"Contrave combines drugs that were used for al...\n","4        \"I have been on this birth control for one cyc...\n","                               ...                        \n","53761    \"I have taken Tamoxifen for 5 years. Side effe...\n","53762    \"I&#039;ve been taking Lexapro (escitaploprgra...\n","53763    \"I&#039;m married, 34 years old and I have no ...\n","53764    \"I was prescribed Nucynta for severe neck/shou...\n","53765                                        \"It works!!!\"\n","Name: review, Length: 53766, dtype: object"]},"execution_count":7,"metadata":{},"output_type":"execute_result"}],"source":["X_test = test_drug_reviews_df['review']\n","X_test"]},{"cell_type":"code","execution_count":18,"metadata":{"colab":{"base_uri":"https://localhost:8080/"},"executionInfo":{"elapsed":290,"status":"ok","timestamp":1679695879799,"user":{"displayName":"Rafaela Queiroz F. Cordeiro","userId":"18015024781067059704"},"user_tz":420},"id":"lG3uzddzmEDW","outputId":"187947d9-f432-4d32-d002-579b9c0ae817"},"outputs":[{"data":{"text/plain":["pandas.core.series.Series"]},"execution_count":18,"metadata":{},"output_type":"execute_result"}],"source":["type(X_test)"]},{"cell_type":"code","execution_count":19,"metadata":{"colab":{"base_uri":"https://localhost:8080/"},"executionInfo":{"elapsed":281,"status":"ok","timestamp":1679695879800,"user":{"displayName":"Rafaela Queiroz F. Cordeiro","userId":"18015024781067059704"},"user_tz":420},"id":"DMbYX-1GmIR7","outputId":"af58cb8f-9e51-4ffd-c99e-1ace37532529"},"outputs":[{"data":{"text/plain":["(53766,)"]},"execution_count":19,"metadata":{},"output_type":"execute_result"}],"source":["X_test.shape"]},{"cell_type":"code","execution_count":8,"metadata":{"colab":{"base_uri":"https://localhost:8080/"},"executionInfo":{"elapsed":1303,"status":"ok","timestamp":1679695880828,"user":{"displayName":"Rafaela Queiroz F. Cordeiro","userId":"18015024781067059704"},"user_tz":420},"id":"1kkxJIMJWdzR","outputId":"4243596d-8176-4726-a644-6dadd7993636"},"outputs":[{"data":{"text/plain":["0        2\n","1        2\n","2        2\n","3        2\n","4        2\n","        ..\n","53761    2\n","53762    2\n","53763    2\n","53764    0\n","53765    2\n","Name: rating_classification, Length: 53766, dtype: int64"]},"execution_count":8,"metadata":{},"output_type":"execute_result"}],"source":["y_test = test_drug_reviews_df.rating_classification\n","y_test"]},{"cell_type":"code","execution_count":21,"metadata":{"colab":{"base_uri":"https://localhost:8080/"},"executionInfo":{"elapsed":169,"status":"ok","timestamp":1679695880833,"user":{"displayName":"Rafaela Queiroz F. Cordeiro","userId":"18015024781067059704"},"user_tz":420},"id":"xxJ51B7zmBiz","outputId":"d08ae58e-fac4-4e39-e043-258ea5586f06"},"outputs":[{"data":{"text/plain":["pandas.core.series.Series"]},"execution_count":21,"metadata":{},"output_type":"execute_result"}],"source":["type(y_test)"]},{"cell_type":"code","execution_count":22,"metadata":{"colab":{"base_uri":"https://localhost:8080/"},"executionInfo":{"elapsed":165,"status":"ok","timestamp":1679695880835,"user":{"displayName":"Rafaela Queiroz F. Cordeiro","userId":"18015024781067059704"},"user_tz":420},"id":"zKsT497KmLDC","outputId":"40a7b39a-d464-4321-e86c-faa66a15a41e"},"outputs":[{"data":{"text/plain":["(53766,)"]},"execution_count":22,"metadata":{},"output_type":"execute_result"}],"source":["y_test.shape"]},{"cell_type":"markdown","metadata":{"id":"nsKamhtDwyN9"},"source":["### Create the models"]},{"cell_type":"markdown","metadata":{"id":"FEVGPDdXELlT"},"source":["#### 3.3 Keras, Tensorflow and Neural Networks"]},{"cell_type":"code","execution_count":9,"metadata":{"colab":{"base_uri":"https://localhost:8080/"},"executionInfo":{"elapsed":1044,"status":"ok","timestamp":1679695881822,"user":{"displayName":"Rafaela Queiroz F. Cordeiro","userId":"18015024781067059704"},"user_tz":420},"id":"bjEoMrO_qajG","outputId":"bfdb9b30-3c06-4b65-9444-9a251b31854c"},"outputs":[{"name":"stderr","output_type":"stream","text":["[nltk_data] Downloading package punkt to\n","[nltk_data]     /Users/rafaelaqueiroz/nltk_data...\n","[nltk_data]   Package punkt is already up-to-date!\n","[nltk_data] Downloading package wordnet to\n","[nltk_data]     /Users/rafaelaqueiroz/nltk_data...\n","[nltk_data]   Package wordnet is already up-to-date!\n"]}],"source":["# Import basic NLP libraries\n","import re # Import regular expressions\n","import nltk # Import nltk packages\n","nltk.download(['punkt', 'wordnet'])\n","from nltk.tokenize import word_tokenize # For tokenization\n","from nltk.stem import WordNetLemmatizer            "]},{"cell_type":"code","execution_count":12,"metadata":{"executionInfo":{"elapsed":6053,"status":"ok","timestamp":1679695895189,"user":{"displayName":"Rafaela Queiroz F. Cordeiro","userId":"18015024781067059704"},"user_tz":420},"id":"yy5QH_Ucu9L2"},"outputs":[],"source":["# Import specifc libraries \n","from sklearn.pipeline import Pipeline\n","from sklearn.feature_extraction.text import CountVectorizer, TfidfTransformer # For BoW and TFIDF\n","\n","from sklearn.metrics import confusion_matrix\n","from sklearn.metrics import classification_report\n","from sklearn.metrics import accuracy_score\n","\n","import keras\n","from keras import layers\n","from keras import models\n","from keras.models import Sequential\n","from keras.layers import Embedding, LSTM, Dense\n","from tensorflow.keras.layers import Dense\n","from tensorflow.keras.utils import to_categorical\n","from keras.preprocessing.text import Tokenizer\n","from keras.utils import pad_sequences\n","\n","import math"]},{"cell_type":"code","execution_count":13,"metadata":{"colab":{"base_uri":"https://localhost:8080/","height":206},"executionInfo":{"elapsed":106,"status":"ok","timestamp":1679695895190,"user":{"displayName":"Rafaela Queiroz F. Cordeiro","userId":"18015024781067059704"},"user_tz":420},"id":"sb4zk1thbzI9","outputId":"ff0bc04f-fc76-4c25-9143-22b2fe745b59"},"outputs":[{"data":{"text/html":["<div>\n","<style scoped>\n","    .dataframe tbody tr th:only-of-type {\n","        vertical-align: middle;\n","    }\n","\n","    .dataframe tbody tr th {\n","        vertical-align: top;\n","    }\n","\n","    .dataframe thead th {\n","        text-align: right;\n","    }\n","</style>\n","<table border=\"1\" class=\"dataframe\">\n","  <thead>\n","    <tr style=\"text-align: right;\">\n","      <th></th>\n","      <th>Unnamed: 0</th>\n","      <th>drugName</th>\n","      <th>condition</th>\n","      <th>review</th>\n","      <th>rating</th>\n","      <th>date</th>\n","      <th>usefulCount</th>\n","      <th>rating_classification</th>\n","    </tr>\n","  </thead>\n","  <tbody>\n","    <tr>\n","      <th>0</th>\n","      <td>163740</td>\n","      <td>Mirtazapine</td>\n","      <td>Depression</td>\n","      <td>\"I&amp;#039;ve tried a few antidepressants over th...</td>\n","      <td>10.0</td>\n","      <td>February 28, 2012</td>\n","      <td>22</td>\n","      <td>2</td>\n","    </tr>\n","    <tr>\n","      <th>1</th>\n","      <td>206473</td>\n","      <td>Mesalamine</td>\n","      <td>Crohn's Disease, Maintenance</td>\n","      <td>\"My son has Crohn&amp;#039;s disease and has done ...</td>\n","      <td>8.0</td>\n","      <td>May 17, 2009</td>\n","      <td>17</td>\n","      <td>2</td>\n","    </tr>\n","    <tr>\n","      <th>2</th>\n","      <td>159672</td>\n","      <td>Bactrim</td>\n","      <td>Urinary Tract Infection</td>\n","      <td>\"Quick reduction of symptoms\"</td>\n","      <td>9.0</td>\n","      <td>September 29, 2017</td>\n","      <td>3</td>\n","      <td>2</td>\n","    </tr>\n","    <tr>\n","      <th>3</th>\n","      <td>39293</td>\n","      <td>Contrave</td>\n","      <td>Weight Loss</td>\n","      <td>\"Contrave combines drugs that were used for al...</td>\n","      <td>9.0</td>\n","      <td>March 5, 2017</td>\n","      <td>35</td>\n","      <td>2</td>\n","    </tr>\n","    <tr>\n","      <th>4</th>\n","      <td>97768</td>\n","      <td>Cyclafem 1 / 35</td>\n","      <td>Birth Control</td>\n","      <td>\"I have been on this birth control for one cyc...</td>\n","      <td>9.0</td>\n","      <td>October 22, 2015</td>\n","      <td>4</td>\n","      <td>2</td>\n","    </tr>\n","  </tbody>\n","</table>\n","</div>"],"text/plain":["   Unnamed: 0         drugName                     condition  \\\n","0      163740      Mirtazapine                    Depression   \n","1      206473       Mesalamine  Crohn's Disease, Maintenance   \n","2      159672          Bactrim       Urinary Tract Infection   \n","3       39293         Contrave                   Weight Loss   \n","4       97768  Cyclafem 1 / 35                 Birth Control   \n","\n","                                              review  rating  \\\n","0  \"I&#039;ve tried a few antidepressants over th...    10.0   \n","1  \"My son has Crohn&#039;s disease and has done ...     8.0   \n","2                      \"Quick reduction of symptoms\"     9.0   \n","3  \"Contrave combines drugs that were used for al...     9.0   \n","4  \"I have been on this birth control for one cyc...     9.0   \n","\n","                 date  usefulCount  rating_classification  \n","0   February 28, 2012           22                      2  \n","1        May 17, 2009           17                      2  \n","2  September 29, 2017            3                      2  \n","3       March 5, 2017           35                      2  \n","4    October 22, 2015            4                      2  "]},"execution_count":13,"metadata":{},"output_type":"execute_result"}],"source":["data = test_drug_reviews_df\n","data.head()"]},{"cell_type":"code","execution_count":14,"metadata":{"colab":{"base_uri":"https://localhost:8080/"},"executionInfo":{"elapsed":2735,"status":"ok","timestamp":1679695897836,"user":{"displayName":"Rafaela Queiroz F. Cordeiro","userId":"18015024781067059704"},"user_tz":420},"id":"QwNqRfbdaGeW","outputId":"64c89425-508b-4b16-e16e-2b5b51d85011"},"outputs":[{"name":"stdout","output_type":"stream","text":["Average length of each review :  42.32192043016496\n","Percentage of reviews with positive sentiment is 66.32036250656553%\n","Percentage of reviews with negative sentiment is 33.67963749343447%\n"]}],"source":["# Check the average lenght of the sentences and, again, the balance between 0 and 2 reviews (this code was adapted from: https://www.analyticsvidhya.com/blog/2022/01/sentiment-analysis-with-lstm/)\n","s = 0.0\n","data = train_medication_reviews\n","for i in data['review_word_lemm']:\n","    word_list = i.split()\n","    s = s + len(word_list)\n","print(\"Average length of each review : \",s/data.shape[0])\n","pos = 0\n","for i in range(data.shape[0]):\n","    if data.iloc[i]['rating_classification'] == 2:\n","        pos = pos + 1\n","neg = data.shape[0]-pos\n","print(\"Percentage of reviews with positive sentiment is \"+str(pos/data.shape[0]*100)+\"%\")\n","print(\"Percentage of reviews with negative sentiment is \"+str(neg/data.shape[0]*100)+\"%\")"]},{"cell_type":"code","execution_count":15,"metadata":{"executionInfo":{"elapsed":4113,"status":"ok","timestamp":1679695901869,"user":{"displayName":"Rafaela Queiroz F. Cordeiro","userId":"18015024781067059704"},"user_tz":420},"id":"YURMjGKxpS97"},"outputs":[],"source":["from sklearn.feature_extraction.text import CountVectorizer\n","\n","vectorizer = CountVectorizer()\n","vectorizer.fit(X_train)\n","\n","X_train_vect = vectorizer.transform(X_train)\n","X_test_vect  = vectorizer.transform(X_test)"]},{"cell_type":"code","execution_count":29,"metadata":{"colab":{"base_uri":"https://localhost:8080/"},"executionInfo":{"elapsed":166,"status":"ok","timestamp":1679695901872,"user":{"displayName":"Rafaela Queiroz F. Cordeiro","userId":"18015024781067059704"},"user_tz":420},"id":"kIL89y46pS7C","outputId":"cb208b12-180a-4113-d5b2-3ab481b3e3a4"},"outputs":[{"data":{"text/plain":["0    ['no', 'side', 'effect', 'take', 'combination'...\n","1    ['son', 'halfway', 'fourth', 'week', 'intuniv'...\n","2    ['used', 'take', 'another', 'oral', 'contracep...\n","3    ['first', 'time', 'using', 'form', 'birth', 'c...\n","4    ['suboxone', 'completely', 'turned', 'life', '...\n","Name: review_word_lemm, dtype: object"]},"execution_count":29,"metadata":{},"output_type":"execute_result"}],"source":["X_train.head()"]},{"cell_type":"code","execution_count":30,"metadata":{"colab":{"base_uri":"https://localhost:8080/"},"executionInfo":{"elapsed":155,"status":"ok","timestamp":1679695901875,"user":{"displayName":"Rafaela Queiroz F. Cordeiro","userId":"18015024781067059704"},"user_tz":420},"id":"kROvf8gX1rwr","outputId":"cbf01322-8f1f-4831-d11c-c9bf7606864e"},"outputs":[{"data":{"text/plain":["(112329,)"]},"execution_count":30,"metadata":{},"output_type":"execute_result"}],"source":["X_train.shape"]},{"cell_type":"code","execution_count":31,"metadata":{"colab":{"base_uri":"https://localhost:8080/"},"executionInfo":{"elapsed":618,"status":"ok","timestamp":1679695902345,"user":{"displayName":"Rafaela Queiroz F. Cordeiro","userId":"18015024781067059704"},"user_tz":420},"id":"kdVHNiRSpS4M","outputId":"d32caf9c-4966-4eaf-9046-997a823b1ee1"},"outputs":[{"data":{"text/plain":["<112329x66660 sparse matrix of type '<class 'numpy.int64'>'\n","\twith 4111730 stored elements in Compressed Sparse Row format>"]},"execution_count":31,"metadata":{},"output_type":"execute_result"}],"source":["X_train_vect"]},{"cell_type":"code","execution_count":32,"metadata":{"colab":{"base_uri":"https://localhost:8080/"},"executionInfo":{"elapsed":180,"status":"ok","timestamp":1679695902350,"user":{"displayName":"Rafaela Queiroz F. Cordeiro","userId":"18015024781067059704"},"user_tz":420},"id":"ORqxfRvi0E0k","outputId":"428b0f11-ad33-4ae5-d387-8cf4151f0b4a"},"outputs":[{"data":{"text/plain":["0    \"I&#039;ve tried a few antidepressants over th...\n","1    \"My son has Crohn&#039;s disease and has done ...\n","2                        \"Quick reduction of symptoms\"\n","3    \"Contrave combines drugs that were used for al...\n","4    \"I have been on this birth control for one cyc...\n","Name: review, dtype: object"]},"execution_count":32,"metadata":{},"output_type":"execute_result"}],"source":["X_test.head()"]},{"cell_type":"code","execution_count":33,"metadata":{"colab":{"base_uri":"https://localhost:8080/"},"executionInfo":{"elapsed":172,"status":"ok","timestamp":1679695902351,"user":{"displayName":"Rafaela Queiroz F. Cordeiro","userId":"18015024781067059704"},"user_tz":420},"id":"KqnwkHYQ1wlO","outputId":"cc8a6c7f-a076-4f37-d907-0a8d2096b2fa"},"outputs":[{"data":{"text/plain":["(53766,)"]},"execution_count":33,"metadata":{},"output_type":"execute_result"}],"source":["X_test.shape"]},{"cell_type":"code","execution_count":34,"metadata":{"colab":{"base_uri":"https://localhost:8080/"},"executionInfo":{"elapsed":166,"status":"ok","timestamp":1679695902352,"user":{"displayName":"Rafaela Queiroz F. Cordeiro","userId":"18015024781067059704"},"user_tz":420},"id":"vu6yYVO_pS0o","outputId":"9e489d81-bcd9-49a5-8e05-c7e1daa96894"},"outputs":[{"data":{"text/plain":["<53766x66660 sparse matrix of type '<class 'numpy.int64'>'\n","\twith 1995631 stored elements in Compressed Sparse Row format>"]},"execution_count":34,"metadata":{},"output_type":"execute_result"}],"source":["X_test_vect "]},{"cell_type":"code","execution_count":42,"metadata":{"executionInfo":{"elapsed":939,"status":"ok","timestamp":1679695903160,"user":{"displayName":"Rafaela Queiroz F. Cordeiro","userId":"18015024781067059704"},"user_tz":420},"id":"jA7Mz1bMpSw3"},"outputs":[],"source":["from keras.models import Sequential\n","from keras import layers\n","\n","input_dim = 66660  # Number of features\n","\n","model = Sequential()\n","model.add(layers.Dense(10, input_dim=input_dim, activation='relu'))\n","model.add(layers.Dense(1, activation='softmax'))"]},{"cell_type":"code","execution_count":43,"metadata":{"colab":{"base_uri":"https://localhost:8080/"},"executionInfo":{"elapsed":62,"status":"ok","timestamp":1679695903162,"user":{"displayName":"Rafaela Queiroz F. Cordeiro","userId":"18015024781067059704"},"user_tz":420},"id":"Z187KKUhpSs8","outputId":"3e57b271-040f-4d2c-d5c3-a286162b9e33"},"outputs":[{"name":"stdout","output_type":"stream","text":["Model: \"sequential_2\"\n","_________________________________________________________________\n"," Layer (type)                Output Shape              Param #   \n","=================================================================\n"," dense_4 (Dense)             (None, 10)                666610    \n","                                                                 \n"," dense_5 (Dense)             (None, 1)                 11        \n","                                                                 \n","=================================================================\n","Total params: 666,621\n","Trainable params: 666,621\n","Non-trainable params: 0\n","_________________________________________________________________\n"]}],"source":["model.compile(loss='categorical_crossentropy', \n","             optimizer='adam', \n","              metrics=['accuracy'])\n","model.summary()"]},{"cell_type":"code","execution_count":44,"metadata":{"executionInfo":{"elapsed":319047,"status":"ok","timestamp":1679696222200,"user":{"displayName":"Rafaela Queiroz F. Cordeiro","userId":"18015024781067059704"},"user_tz":420},"id":"F7RbhnbtpSod"},"outputs":[],"source":["# Import 'to_categorical module\n","from keras.utils import to_categorical\n","\n","# Convert y_train and y_test to one-hot encoded arrays\n","y_train_onehot = to_categorical(y_train, num_classes=3)\n","y_test_onehot = to_categorical(y_test, num_classes=3)\n","\n","# Convert the one-hot encoded arrays to arrays of shape (None, 1)\n","y_train_onehot_1d = np.argmax(y_train_onehot, axis=-1).reshape(-1, 1)\n","y_test_onehot_1d = np.argmax(y_test_onehot, axis=-1).reshape(-1, 1)\n","\n","# Train the model using the 1D arrays\n","history = model.fit(X_train_vect, y_train_onehot_1d,\n","                    epochs=15,\n","                    verbose=False,\n","                    validation_data=(X_test_vect, y_test_onehot_1d),\n","                    batch_size=10)"]},{"cell_type":"code","execution_count":45,"metadata":{"colab":{"base_uri":"https://localhost:8080/"},"executionInfo":{"elapsed":1447,"status":"ok","timestamp":1679696223601,"user":{"displayName":"Rafaela Queiroz F. Cordeiro","userId":"18015024781067059704"},"user_tz":420},"id":"SW98j17e86PB","outputId":"50c3e3fb-e470-4af7-d7b7-fcd508a86325"},"outputs":[{"name":"stdout","output_type":"stream","text":["Training Accuracy: 0.0890\n","Testing Accuracy:  0.0898\n"]}],"source":["loss, accuracy = model.evaluate(X_train_vect, y_train_onehot_1d, verbose=False)\n","print(\"Training Accuracy: {:.4f}\".format(accuracy))\n","loss, accuracy = model.evaluate(X_test_vect, y_test_onehot_1d, verbose=False)\n","print(\"Testing Accuracy:  {:.4f}\".format(accuracy))"]},{"cell_type":"code","execution_count":46,"metadata":{"executionInfo":{"elapsed":16,"status":"ok","timestamp":1679696223607,"user":{"displayName":"Rafaela Queiroz F. Cordeiro","userId":"18015024781067059704"},"user_tz":420},"id":"E0tg4RdOpSfA"},"outputs":[],"source":["import matplotlib.pyplot as plt # This code was adapted from this site: https://realpython.com/python-keras-text-classification/\n","plt.style.use('ggplot')\n","\n","def plot_history(history):\n","    accuracy = history.history['accuracy']\n","    val_accuracy = history.history['val_accuracy']\n","    loss = history.history['loss']\n","    val_loss = history.history['val_loss']\n","    x = range(1, len(accuracy) + 1)\n","\n","    plt.figure(figsize=(12, 5))\n","    plt.subplot(1, 2, 1)\n","    plt.plot(x, accuracy, 'b', label='Training acc')\n","    plt.plot(x, val_accuracy, 'r', label='Validation acc')\n","    plt.title('Training and validation accuracy')\n","    plt.legend()\n","    plt.subplot(1, 2, 2)\n","    plt.plot(x, loss, 'b', label='Training loss')\n","    plt.plot(x, val_loss, 'r', label='Validation loss')\n","    plt.title('Training and validation loss')\n","    plt.legend()"]},{"cell_type":"code","execution_count":47,"metadata":{"colab":{"base_uri":"https://localhost:8080/","height":337},"executionInfo":{"elapsed":1120,"status":"ok","timestamp":1679696224711,"user":{"displayName":"Rafaela Queiroz F. Cordeiro","userId":"18015024781067059704"},"user_tz":420},"id":"Fq-dqktEpSa4","outputId":"5f1449de-e23c-4c74-8ea0-3ec214ab627b"},"outputs":[{"data":{"image/png":"iVBORw0KGgoAAAANSUhEUgAAA/AAAAHECAYAAACeK8GcAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjUuMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8qNh9FAAAACXBIWXMAAA9hAAAPYQGoP6dpAAB3pUlEQVR4nO3deVgW9f7/8dfNooDIYkgoaAKCW+5rYm4kHpfMPbeTaFamlnYqy/yqWamHTqllZS6l1knFNHdTcim3b5oelZOaomipiYqIqICAzO8Pf9zfblkERWHw+bgur4v7M5+Zeb/nRmbes3zGYhiGIQAAAAAAUKzZFXUAAAAAAADg9ijgAQAAAAAwAQp4AAAAAABMgAIeAAAAAAAToIAHAAAAAMAEKOABAAAAADABCngAAAAAAEyAAh4AAAAAABOggAcAAAAAwAQo4FHoqlSpoipVqtz1csLDw2WxWHTy5Mm7XlZJUljbt7C0bt1aFovFpu3HH3+UxWLR22+/ne/lvP3227JYLPrxxx8LN8Bb5BQvAKB44pji3uKY4u4Ux2OK4vadovBRwJdAFoulQP/mz59f1CEDhYaDNAAoPBxT4EHGMQWKI4eiDgCFb8KECdnapk+frsuXL2vkyJHy8PCwmVavXr1CXf+mTZsKZTlTpkzRm2++KV9f30JZHu6fJk2a6PDhw/Ly8irqULL56quvlJycXNRhAIApcEyBosYxBWCLAr4EyukWo/nz5+vy5csaNWrUPb+tJjAwsFCWU6FCBVWoUKFQloX7y8XFRdWrVy/qMHJUuXLlog4BAEyDYwoUNY4pAFvcQv+Ay7o1KDY2VtOnT1ft2rXl7Oys1q1bS5LS0tL0ySefqGPHjnrkkUdUunRpeXp6KjQ0VGvXrs1xmTk9ezN//nzrrXVbtmxR69atVbZsWbm5ualjx446ePBgrrH99balkydPymKxKDw8XCdPnlSfPn3k5eUlJycnNWzYUKtWrcoxpqwDDT8/Pzk5Oal69eqaOnWqYmNjrcvLj7vZHsnJyXr99ddVuXJllS5dWlWrVtU///lPGYaRbR7DMPTJJ5+oVq1acnJykq+vr4YPH67Lly/nK05JOnPmjOzt7dWgQYNc+zzxxBOyWCz69ddfrW3z589Xjx49FBAQIGdnZ7m5uSkkJERfffVVvted1/Nqe/fu1d/+9jfr9//EE09o586duS5rxYoVGjBggIKDg1WmTBm5urqqQYMGmj59um7cuGHT12KxaMGCBZIkf39/6y2df/19zO15tczMTH322Wdq3LixXF1dVaZMGTVq1EifffaZMjMzs/W3WCxq3bq14uPj9fzzz6tChQoqXbq0atWqpS+++CKfW+qmLVu26Pnnn1fNmjXl5uYmZ2dn1apVSxMmTFBKSkqO89y4cUOff/65QkJC5O7uLmdnZ1WtWlVDhgxRTEzMHfXN61bB3L7TrO15/fp1jR8/XkFBQSpVqpT1/9Tly5f1r3/9S23btpWfn59KlSql8uXLq0uXLnl+77/99psGDx6sKlWqqHTp0vL29tbjjz+umTNnSpIuXbokFxcXBQYG5vh/SJI6d+4si8WivXv35roeAIWHYwqOKTimuKkojylyk5qaqilTpqh27dpycXGRm5ubHn/8cS1evDjH/suXL1ebNm3k4+Oj0qVLy8fHRy1atNCnn35q0+/YsWMaMmSIAgMD5eTkJE9PT9WoUUMvvPCCLl68WCixwxZX4CFJevnll7V9+3Z16tRJHTt2lL29vSQpISFBI0eOVPPmzdWuXTuVL19eZ8+e1cqVK9W5c2fNmjVLzz//fL7Xs2bNGq1cuVIdOnTQ0KFDdejQIa1bt06//PKLDh06pPLly+drOb///ruaNGmigIAA/f3vf1dCQoIiIyPVtWtX/fDDDwoNDbX2TU1NVdu2bfWf//xH9evXV//+/XX58mVNmjRJ27ZtK9B2utPtkZ6errCwMP3555/q0KGDHBwctGLFCo0ZM0YpKSmaOHGiTf9Ro0bp448/VoUKFfT888/L0dFRK1eu1O7du5WWlqZSpUrdNlZfX1898cQTioqK0n//+1/Vrl3bZvrp06e1ZcsWNWzYUI8++qi1/cUXX1TNmjXVsmVLVahQQfHx8Vq7dq0GDhyo3377TZMnTy7QNvurnTt36oknnlBaWpq6d++uqlWrav/+/WrTpo3atm2b4zxvvvmm7Ozs1LRpU/n6+ioxMVGbNm3SK6+8ot27d2vhwoXWvhMmTNCKFSt04MABm1s7b73FMyf9+vVTZGSkKleurCFDhshisWj58uUaPny4tm7dmuMOLjExUSEhISpVqpR69uyp1NRULV26VEOGDJGdnZ0GDRqUr+0SERGh3377Tc2bN1enTp2UkpKiHTt26J133tGWLVu0efNmOTj835/rtLQ0derUSRs3blSlSpXUv39/lS1bVidPntTy5cvVokULBQUFFbjv3ejRo4f27NmjDh06qGvXrnr44YclSYcPH9bYsWPVsmVLderUSZ6envr999+1cuVKrVu3TqtWrVLHjh1tlrV27Vr16tVL169f19/+9jf17dtXly5d0oEDB/T+++/rxRdflKenp/r06aN58+Zp48aNateunc0yTp06pe+//14NGzZUw4YN7zo/APnHMUX+cEzBMcVfFdYxRU7S0tIUFhambdu2qWbNmho+fLiSk5P17bffqm/fvtq3b58iIiKs/WfOnKlhw4bJx8dHXbp0kZeXl86fP6/o6GjNnz9fw4cPlyT9+eefatKkia5cuaKOHTta4z5x4oT+/e9/66WXXtJDDz10x3EjFwYeCI888oghyThx4oRN+8CBAw1JRsWKFY3Y2Nhs86WmphqnTp3K1p6QkGDUqFHD8PT0NJKTk7Ot65FHHrFpmzdvniHJsLe3NzZu3Ggz7c033zQkGf/85z9zjO2vMZ84ccKQZEgy3n77bZv+69evNyQZf/vb32za33nnHUOS0adPHyMzM9Pa/scffxheXl6GJGPgwIHZcszJnW4PSUaHDh1spp07d85wd3c33NzcjLS0NGv7jh07DElGYGCgcfHiRWt7SkqK0axZM0NStu2bm2+++caQZLz66qvZpk2ZMsWQZHz88cc27ceOHcsx79atWxsODg7Z8m/VqpVx65+SLVu2GJKMCRMmWNsyMzONatWqGZKMFStW2PSfPn269XvdsmXLbeO5ceOG0b9/f0OS8b//+78203L6vbldvFnbqVGjRsbVq1et7VevXjUaNGhgSDL+/e9/28yTFe+zzz5rZGRkWNsPHjxo2NvbG9WrV89x/Tk5fvy4ze9mljFjxhiSjEWLFuXY/uSTTxqpqak201JTU43z58/fUd+8tl1O36lh/N/2rF27tnHhwoVs8yUmJubYfvLkSePhhx82qlWrZtN+4cIFw83NzXB0dDR++umnbPP98ccf1p/37NljSDJ69OiRrd+4ceMMScbs2bOzTQNwdzim4JjiVhxT/J+iPqbI6f/MpEmTDElG586djfT0dGt7XFycUalSJUOSsW3bNmt7/fr1jVKlShnnzp3Ltvy/7tM/+ugjQ5Ixbdq0bP2uXr2a7fcXhYMC/gFxu51tTv/xbueDDz4wJGU7yM5rZztgwIBsy4mNjc3xIDyvnW2VKlVs/sBlqVy5svHQQw/ZtAUGBhp2dnY5/vF97733CrSzzUte20NSjjuNZ555xpBk/Pe//7W2DRkyxJBkfPnll9n6Z+3E8ruzTU5ONtzc3AwfH59s26tGjRqGo6NjjsVVTpYuXWpIMhYsWGDTnt+d7fbt2w1JRsuWLbMtOyMjwwgMDMxxZ5ubrOJt4sSJNu13srMNDQ01JBk//PBDtv5RUVGGJKNNmzY27ZIMFxcXIykpKds8LVu2NCTlOK0g4uPjDUnGoEGDrG0ZGRmGu7u74ezsbJw5cybP+QvS1zDuroBfvnx5flKyMWLECEOS8fvvv1vbsv4fvfzyy/laRuPGjQ1HR0cjLi7O2paRkWFUrFjRKFu2rM3BE4DCwTEFxxQcU+Qeb1EfU+T0fyYwMNCwWCzGkSNHsvWfPXt2tmONBg0aGC4uLkZCQkKe6/r4448NScasWbPyFRsKB8/AQ5LUtGnTXKcdPHhQ4eHh1ueXsp4Beu211yTdfC4qvxo1apStrVKlSpJuPtOaX/Xq1bPeknfrsv66nKSkJB0/fly+vr45DrTTokWLfK8zy51sDw8PjxwH4skp9//85z+SpFatWmXr//jjj9vcSn07zs7O6t27t+Li4rRhwwZr+y+//KLDhw+rc+fO2UZ1/eOPPzR8+HBVr15dLi4u1vx69uyZa375kVde9vb2uX4XFy9e1Jtvvqk6derI1dXVGk/W79KdxvNX+/btk52dXY6xtWnTRvb29tb4/yo4OFhly5bN1p71vSYmJuZr/deuXdPkyZPVuHFjubu7y87OThaLxfrd/DXH3377TZcvX1adOnVUsWLFPJdbkL53K6+/ITt27FDv3r1VqVIllS5d2vodfvLJJ5Js8/v5558lSR06dMjXeocNG6b09HR9+eWX1rbVq1frzz//1IABA1SmTJk7SQfAXeCYIv84puCYIkthHVPc6sqVK9bf2+Dg4GzTn3jiCUmyial///5KTk5WrVq19I9//EMrVqzQhQsXss3bpUsXubq6avjw4erVq5dmz56tgwcP5jo2DQoHz8BDkuTj45Nj+88//6y2bdsqIyNDoaGh6tKli9zc3GRnZ6f9+/dr5cqVun79er7X4+7unq0ta+dx6+AhBV1O1rL+OjhIUlKSJFmfx71Vbu25udPtkVe8km3uWYPK5BSbvb19gZ8lGjhwoObOnasFCxZYnzXOGpRl4MCBNn1jY2PVpEkTXbp0SY8//rjCwsLk7u4ue3t7nTx5UgsWLCjQ9/1XeeUl5fw7mJiYqMaNG+vEiRNq0qSJnnnmGZUrV04ODg5KTEzURx99dMfx3BpbuXLl5OjomG2ag4OD9dmvWxXke81Nenq62rZtq927d+vRRx/V008/rfLly1tjmThxok2OWTvw/LwKqSB971Zuf0OWL1+unj17ysnJSe3atVNgYKDKlCkjOzs7/fjjj/rpp5/uOD9Jevrpp/Xqq69q7ty5evPNN2WxWDRr1ixJ0tChQ+8uKQB3hGOK/OGYgmOKvyqMY4rc4pFy/3+Z9XaGvw5q+I9//ENeXl767LPP9NFHH2natGmyWCxq06aN/vWvf1kHM3zkkUe0e/duvf3221q/fr2WLl0q6eZJh9GjR2vEiBF3FDPyRgEPScpxBE1Jeu+995SSkmId5fWvpkyZopUrV96H6O6cm5ubJOncuXM5Ts+tPTf3Y3tk/QE/d+6cAgICbKbduHFDFy9eLFBB1qJFCwUGBmrlypVKTExUmTJltHjxYnl5eWUbPGzq1Km6ePGi5s2bl20U3UWLFll30nfir3nlJC4uLlvb3LlzdeLECU2YMCHb6LP/+7//q48++uiO47k1toSEBKWnp2fb4WZkZCg+Pt76u1TYsgYSGjhwoObPn28z7ezZs9kGI8oaPCc/VwkK0leS7Oxu3pSVkZGRbdrtzvzn9jdk3LhxKlWqlPbs2aMaNWrYTHvhhRf0008/5RrzrYMk5cTZ2Vnh4eGaOnWqfvjhBwUHBysqKkrNmjVTnTp1bjs/gMLHMUX+cEzBMcX9kLWtctom0s1jjb/2y/LMM8/omWeeUWJionbu3Knly5fryy+/VFhYmA4fPmwdJLJGjRqKjIxURkaGDhw4oI0bN2rGjBl66aWXVKZMmbsafA854xZ65OnYsWMqV65cth2LpGwH3sWRm5ubAgICdObMmRxfjbV9+/YCLe9+bI+ss5o5LW/btm05Fle388wzz+j69euKjIzUmjVrdPHiRfXr1y/bjuXYsWOSbo4ofqu7zS+vvG7cuJHjd3En8WTdBlmQM9X169dXZmamtm7dmm3a1q1bdePGjTxfnXM3Cppj9erV5eHhoejoaOtONzcF6StJnp6ekm6O4H6rPXv23Hb+nBw7dkw1a9bMVrxnZmbm+J03a9ZMkmxuz7ydF1980Xrlfc6cOcrMzNQLL7xwR/ECuHc4prDFMcWd45gi/8qWLavAwECdOXMm22tmpZuvspWUa0weHh7q2LGj5syZo/DwcF28eDHHNy44ODioYcOGeuONN7Ro0SJJN+/CQ+GjgEeeqlSpooSEBEVHR9u0f/HFFwU6wC5KzzzzjDIzMzVmzBibZ3JOnTql6dOnF2hZ92N7ZJ2lnjRpkhISEqztqampGjNmzB0tc+DAgbJYLPrqq6+s717N6T21Wc/0Zf0xz7JhwwbNnTv3jtadpXnz5qpWrZq2bt2a7arCJ598ouPHj+c7nn379mnKlCk5rifrdsCcitDcDB48WJI0ZswYJScnW9uTk5P15ptvSpKeffbZfC+vIHLLMTY2Vm+88Ua2/vb29ho2bJhSUlI0bNgwpaWl2UxPS0uzPqdWkL7S/z23OmfOHJt+//3vf+/4ykSVKlUUExNjcxeAYRiaOHGiDh06lK3/wIED5ebmps8++0w7duzINv306dPZ2qpWrap27dpp1apVmj17tjw8PPT000/fUbwA7h2OKWxxTHHnOKYomMGDB8swDL3++us2JyPi4+P17rvvWvtkWb9+fY4nd7Ju/XdycpIk7d69O8e7ILLasvqhcHELPfI0atQobdiwQS1atFDv3r3l7u6uPXv2aPv27erZs6f1WZfibPTo0VqxYoUWL16sI0eOKCwsTJcvX9aSJUvUsmVLrVixwnrr8O3cj+0REhKil156STNmzNCjjz6qnj17Wt/Z6unpaX1WqSAeeeQRtWrVSj/++KMcHBxUu3Zt1a9fP1u/YcOGad68eerdu7d69OghX19f/frrr1q/fr169+6tyMjIO87LYrHoiy++ULt27dSjRw/rO1uzbrf629/+pvXr19vM88wzz+hf//qXXnnlFf34448KCgpSTEyM1qxZo+7du+cYT2hoqP71r3/pueeeU48ePeTq6ioPD488n8Pq16+fVq5cqSVLlqhWrVrq2rWrLBaLVqxYoRMnTqh3797q37//HeeelyeffFJVq1bVtGnT9Ouvv6p+/fr6448/tGbNGnXq1El//PFHtnkmTJigXbt2acWKFQoODlanTp1UtmxZnTp1SlFRUfrXv/5lPZgqSN+nnnpK1apV06JFi3T69Gk1bdpUf/zxh1auXKmnnnpKS5YsKXB+r7zyioYOHaoGDRqoR48ecnR01I4dO3To0CE9+eSTWr16tU1/Ly8vLVy4UD179lTLli3VsWNH1a5dW5cvX1Z0dLROnz6tEydOZFvPiy++qKioKMXHx+vll1+Ws7NzgWMFcG9xTGGLYwqOKe6X1157Td9//71WrlypunXrqmPHjtb3wJ8/f16jR4+2GfivT58+cnJyUosWLVSlShUZhqFt27bpl19+UYMGDawD3y1cuFCffvqpWrVqpapVq8rT01PHjx/X6tWrVbp0aY0cOfK+5vnAKMoh8HH/3O6VL7m9HsMwDGP16tVG06ZNDVdXV8Pd3d1o166d8dNPP1lf4zJv3rxs68rtlS+39s0iyWjVqtVtY8t65Utur2jJ6XUehmEYly5dMl566SWjQoUKRqlSpYxq1aoZH3zwgbFr1y5DkjFq1Khc879VYWyPLBMmTMjxNSeZmZnGjBkzjOrVqxulSpUyKlSoYAwbNsxITEzMc3l5yYpPkvHBBx/k2m/Hjh1GmzZtDA8PD8PV1dUICQkxli9fftvXiP1Vbn0N4+arWtq3b2+4uroarq6uRmhoqLFz585ct8XBgweNJ5980ihfvrzh4uJiNGjQwJgzZ06evwsffvihddvpllfk5PY7cuPGDePTTz81GjZsaDg7OxvOzs5GgwYNjE8++cS4ceNGtv45/c5myc//q7/6448/jH79+hkVK1Y0nJycjJo1axoRERFGenp6rutJT083ZsyYYTRu3NgoU6aM4eLiYlStWtV47rnnjJiYmDvue/r0aePpp582PD09DScnJ6NRo0bGsmXLCvT932revHlG3bp1DRcXF+Ohhx4yunbtakRHR+f6nRuGYfz666/G3//+d6NixYqGo6Oj4e3tbbRs2TLXV9VkZGRY38F88ODBPOMBcHc4puCYgmOK3OM1jKI9psjtO01JSTEmTZpk1KpVy3BycrJ+HwsXLszWd+bMmUbXrl0Nf39/w9nZ2fD09DTq1atnRERE2LzO7ueffzaGDh1q1KlTx3rcEBgYaISHh9u8zhCFy2IYjPOPB9ecOXP0/PPP6/PPP+eZWcDEjh8/rqCgILVo0SLH5w4B4F7jmALA/cAz8Hgg/Pnnn9naTp06pXfffVeOjo7q0qVLEUQFoLD861//kmEYvLIGwD3HMQWAosQz8Hgg9OjRQ+np6WrYsKE8PDx08uRJrVmzRsnJyXr//ffv6BkwAEXr999/19dff62YmBh9/fXXql+/vnr27FnUYQEo4TimAFCUuIUeD4SZM2fqm2++0dGjR3Xp0iW5urqqQYMGeumll9S1a9eiDg/AHfjxxx/Vpk0blSlTRo8//rhmzpxpHWUYAO4VjikAFCUKeAAAAAAATIBn4AEAAAAAMAEKeAAAAAAATIACHgAAAAAAE6CABwAAAADABHiNXA4uXbqkjIyMog6jwMqXL68LFy4UdRj3BLmZE7mZE7kVLw4ODvL09CzqMEokM+7vzfg7nF/kZk7kZk7kVrwUZF9PAZ+DjIwMpaenF3UYBWKxWCTdjL2kvViA3MyJ3MyJ3PAgMdv+viT/DpObOZGbOZGbuXELPQAAAAAAJkABDwAAAACACVDAAwAAAABgAhTwAAAAAACYAIPYAQAAAEAOrl69qoyMDOvgaCVFSkqK0tLSijqMe6I45+bi4iIHh7srwSngAQAAAOAWiYmJslgscnd3L+pQCp2jo6Op3sJREMU1t8zMTF25ckVlypS5qyKeW+gBAAAA4BaXLl2Ss7NzUYeBEsLOzk5ly5ZVcnLy3S2nkOIBAAAAgBLDMIwSd+s8ipad3d2X3xTwAAAAAACYAAU8AAAAAAAmQAEPAAAAAMhRz549NX78+Hz3P3XqlHx9ffXrr7/ew6iknTt3ytfXV5cvX76n6yluGIUeAAAAAEzO19c3z+m9evXS9OnTC7zcOXPmyNHRMd/9K1asqH379qlcuXIFXhdujwIeAAAAAExu37591p9XrVqlDz74QFu3brW2OTk52fRPT0/PV2Hu6elZoDjs7e3l7e1doHmQfxTw94phyJKSct9WZ7FYpGvXZElOlgzjvq33fiA3cyI3cyK3wmU4O0uMYAwAuA/+WjSXLVtWFovF2nbq1CnVr19fM2fO1FdffaX//Oc/mjJlitq1a6f/+Z//0a5du5SYmKgqVaropZdeUteuXa3L6tmzp2rWrKl33nlHktS0aVP1799fJ0+e1Jo1a+Tu7q6RI0dqwIAB1nU1a9ZMGzZs0KOPPqqdO3eqV69eWrx4sSZPnqyjR4+qVq1amjp1qqpWrWpdz/Tp0/Xll18qNTVVXbp0Ubly5bRlyxb98MMP+d4Ga9eu1YcffqgTJ07I29tbgwYN0tChQ63T58+frzlz5ujs2bMqW7asmjRpojlz5kiS1qxZo2nTpunkyZNycnLSo48+qnnz5snFxaXgX8Y9RAF/j1hSUlQhKOi+r9fnvq/x/iE3cyI3cyK3wnE2JkZGMdvxAwAKzjCklJSiOSHr7GwU2rngyZMna/z48fr4449lZ2en69evq06dOho2bJjKli2rTZs26eWXX1blypXVoEGDXJcza9Ysvf7663rppZe0du1ajRkzRs2aNbMpyG8VERGh8ePH66GHHtKbb76pV199VStXrpQkfffdd5oxY4YmT56sRo0aadWqVZo1a5YqVaqU79yio6M1dOhQvf766+rUqZP27Nmjt956S56ennr66ad14MABa+6NGjVSYmKidu3aJUk6d+6chg8frrFjx6pDhw66evWqdu3aJaMYXsyggAcAAACAPKSkWBQUVKFI1h0Tc1YuLoVTSA4ZMkQdO3aUo6Oj0tPTJcnmCvXgwYO1ZcsWrVmzJs8Cvm3btgoPD5ckDR8+XHPmzNHOnTvzLODfeOMNPfbYY9Z5nnnmGaWmpsrJyUlffvml+vTpo6efflqS9Morr+inn37StWvX8p3b7Nmz1aJFC7366qtKT09XYGCgYmJi9Pnnn+vpp5/WmTNn5OLioieeeEKurq7y8/PTo48+Kkk6f/68MjIy1LFjR/n5+UmSatSoke91308U8PeI4eysszEx9219FotFPj4+iouLK5Zniu4GuZkTuZkTuRUuw9n5vqwHAID8qFu3rs3nGzdu6JNPPtHq1at19uxZpaWlKS0t7ba3jdesWdP6s8ViUfny5XXx4sV8z/Pwww9Lki5evChfX1/FxsZq4MCBNv3r1aunHTt25CsvSYqJiVH79u1t2ho3bqy5c+fqxo0batmypfz8/PTYY4+pdevWatOmjTp06CBnZ2fVrFlTLVq0UGhoqFq1aqVWrVqpU6dO8vDwyPf675c7KuA3bNigVatWKTExUX5+fgoPD8/zDMWhQ4e0YMECnT59Wp6enurSpYvCwsJs+qxdu1ZRUVGKj4+Xm5ubmjZtqn79+qlUqVKSpJSUFEVGRmr37t26fPmy/P39FR4ebnOWJzU1Vd98841++eUXXblyRd7e3urQoUO2dd0XFsv9vW3SYpHKlJHh4lLiDrrJzaTIzZzIDQCAbJydDcXEnC2ydRfesmxPLM+aNUtz5szRxIkTVb16dbm4uGjChAnWq/O5cXCwLSMtFosyMzMLNI8km3kstzwnUNB9tWEYeS7D1dVV69ev186dO7V161Z98MEH+vDDD7Vu3Tq5u7tr8eLF2rNnj3766SfNmzdPERERWrNmjSpXrlygOO61AhfwO3fu1Pz58zVkyBBVq1ZNGzdu1OTJkzVt2jR5eXll63/+/HlNmTJFoaGheumll3TkyBHNnTtXbm5uatasmSRp27ZtWrhwoV588UUFBwfr7Nmz+uyzzyTJemvG559/rlOnTmnEiBEqV66ctm7dqnfffVfTpk2zvqJg/vz5OnjwoF566SWVL19e0dHRmjt3rjw9PdW4ceM73UYAAAAAHmAWiwrtNvbiZNeuXWrfvr169Ogh6WZBfeLECQXd57G8AgICtH//fvXs2dPaFh0dXaBlBAcHa/fu3TZte/bsUUBAgOzt7SXdPInQsmVLtWzZUv/4xz9Uo0YN7dixQx07dpTFYlHjxo3VuHFjvfLKK2rSpIm+//57vfDCC3efYCGyK+gMa9asUdu2bRUaGmq9+u7l5aWoqKgc+0dFRcnLy0vh4eHy8/NTaGio2rRpo9WrV1v7HD16VNWqVVOLFi3k7e2tunXrKiQkRLGxsZKktLQ07dq1SwMGDFDNmjXl4+Oj3r17y9vb22a9MTExatWqlWrVqiVvb2898cQTeuSRR3T8+PGCpgkAAAAAJVqVKlW0detW/fLLL4qJidEbb7yhCxcu3Pc4Bg8erEWLFmnJkiWKjY3V9OnTdfjw4WxX1PPywgsvaPv27frwww91/PhxLVmyRPPmzbMW4D/88IO++OIL/frrrzp9+rS+/fZbZWZmKjAwUP/5z3/08ccf68CBAzpz5ozWrVunhISE+34iIz8KdAU+IyNDsbGxNq8VkKQ6deroyJEjOc4TExOjOnXq2LTVq1dPW7ZsUUZGhhwcHFS9enVt27ZNx44dU9WqVXXu3Dnt27dPrVq1knTz2YzMzMxs7yksVaqUfvvtN+vnatWqae/evWrbtq08PT118OBBnT17VoMGDcoxtvT0dJvbQywWi/W2koL8shQHWfGaLe78IDdzIjdzIjcAAB4co0aN0qlTp9S/f385Ozurf//+at++va5cuXJf4+jevbt+//13vfvuu7p+/bqefPJJ9e7d2+bd9rdTu3Ztff755/rwww81depUeXt76/XXX7cOjOfu7q7vv/9eU6dOVWpqqvz9/fXpp5+qWrVqiomJ0a5duzR37lxdvXpVvr6+Gj9+vNq2bXuvUr5jFqMADxckJCRo6NChevfdd1WtWjVr+3fffaeffvpJH330UbZ5Ro4cqVatWql79+7WtiNHjmjcuHGaNWuWPD09JUnff/+9vvrqK0k3C/awsDANGTLEOs///M//yMHBQS+//LI8PDy0fft2ffrpp/Lx8bGuNyMjQ59//rm2bt0qe3t7WSwWDR06VC1btswxnyVLlmjp0qXWz/7+/oqIiMjv5gAAACZ24cKF2z7nWZxYLBZVqFBBZ8+eLXHjOJCbOZX03FJSUuTk5FTUodwTfx2Fvrjq06ePypcvrxkzZhRovuKeW1JSktzc3GzaHB0dVb58+XzNf0eD2OV09SKvKxq5DSaQ1X7w4EF99913GjJkiIKCghQXF6d58+bJw8PD+hzEiBEjNHPmTA0dOlR2dnby9/dXSEiITpw4YV3uunXrFBMTo9GjR6t8+fI6fPiw5s6dKw8Pj2x3AUhSt27d1Llz52xxXrhwQRkZGfndHMUCI0ebE7mZE7mZk1lzc3BwyPdOHQAAM0pJSdFXX32l1q1by97eXitWrNC2bdu0aNGiog6t2ClQAe/m5iY7OzslJibatF++fFnu7u45zuPh4ZGtf1JSkuzt7eXq6ipJioyMVMuWLRUaGipJqly5slJTUzV79mx1795ddnZ28vHx0cSJE5WamqqUlBR5enpq2rRp8vb2lnTzOflFixbp9ddft76z8JFHHtHJkye1evXqHAt4R0fHbLflZzHTwd1fGYZh2thvh9zMidzMidwAAMD9tHnzZn388ce6fv26AgMDNWfOnFzvpH6QFaiAd3BwUEBAgKKjo9WkSRNre3R0dK6jvAcFBWnv3r02bQcOHFBAQID1VQLXr1/PdpXezs4uxwMsJycnOTk56erVqzpw4IAGDBgg6ebt8zdu3Mj3cgAAAAAARc/Z2VmRkZFFHYYpFPgW+s6dO2vGjBkKCAhQcHCwNm7cqPj4eLVr106StHDhQiUkJGjEiBGSpLCwMG3YsEELFixQaGiojh49qs2bN2vkyJHWZTZs2FBr166Vv7+/9Rb6yMhINWrUSHZ2NwfK379/vySpYsWKiouL09dff62KFSuqdevWkiQXFxfVrFlT//73v1WqVCmVL19ehw4d0k8//aSBAwfezTYCAAAAAKDIFbiAb968ua5cuaJly5bp0qVLqlSpksaMGWN9Pu/SpUuKj4+39vf29taYMWO0YMECbdiwQZ6enho0aJD1HfCS1KNHD1ksFi1evFgJCQlyc3NTw4YN1bdvX2uf5ORkLVq0SBcvXpSrq6uaNm2qvn37Wq/iSzdHUVy4cKE+/vhjXb16VeXLl1ffvn2tJxcAAAAAADCrAo1C/6Aw26i0UskfBZTczIfczIncip+CjEyLgjHb/t6sv8P5QW7mVNJzYxR6cyruud3tKPR29yIoAAAAAABQuCjgAQAAAAAwAQp4AAAAAABMgAIeAAAAACBJ6tmzp8aPH2/93LRpU82ZMyfPeXx9fbV+/fq7XndhLScvH374oakHOaeABwAAAACTGzhwoJ5++ukcp+3Zs0e+vr7673//W+Dlrlu3TgMGDLjb8GzkVkTv27dPbdq0KdR1lTQU8AAAAABgcn379tWOHTt0+vTpbNMiIyNVq1Yt1a5du8DLfeihh+Ts7FwYId6Wt7e3SpcufV/WZVYU8AAAAABgck888YS8vLy0ZMkSm/aUlBStWrVKffv2VUJCgoYNG6a6desqMDBQoaGhWrFiRZ7LvfUW+tjYWHXv3l0BAQFq3bq1tm7dmm2eSZMmqUWLFgoMDNRjjz2m999/3/pqt8jISE2dOlWHDh2Sr6+vfH19FRkZKSn7LfSHDx9Wr169FBgYqFq1amn06NG6du2adfqoUaM0ePBgff7556pfv75q1aqlN954o0CvkcvMzNS0adPUsGFD+fv7q127dtqyZYt1elpamsaOHav69esrICBATZs21YwZM6zTP/zwQzVu3Fj+/v5q0KCBxo0bl+913wmHe7p0AAAAADA7w5AlJaVoVu3sLFkst+3n4OCgnj17asmSJXrllVdk+f/zrF69Wunp6erWrZtSUlJUp04dvfzyy3J2dtamTZv08ssvq3LlymrQoMFt15GZmannnntOnp6eWrVqla5evaoJEyZk61emTBlNmzZNPj4+Onz4sEaPHi1XV1cNGzZMXbp00ZEjR/Tjjz9q8eLFkqSyZctmW0ZKSooGDBigBg0aaO3atYqPj9frr7+usWPHavr06dZ+O3fulLe3t7799ludOHFCw4YNU40aNdS/f//b5iNJc+fO1axZsxQREaFatWopMjJSgwYN0ubNmxUQEKAvv/xSUVFR+vzzz+Xr66s///xTf/75pyRpzZo1mjNnjj777DNVq1ZN58+f16FDh/K13jtFAQ8AAAAAebCkpKhCUFCRrPtsTIwMF5d89e3Tp49mzpypnTt3KiQkRNLNK94dOnSQh4eHPDw8NHToUDk6Oio9PV2DBw/Wli1btGbNmnwV8Nu2bVNMTIx+/vlnVaxYUZL05ptvZntGftSoUdafK1WqpOPHj2vVqlUaNmyYnJ2dVaZMGdnb28vb2zvXdX333XdKTU3VRx99JJf/n/97772n8PBwjR07VuXLl5ckubu7a9KkSbK3t1fVqlX1xBNPaPv27fku4GfNmqVhw4bpqaeekiSNHTtWO3fu1Ny5czV58mSdOXNG/v7+atKkiSwWi/z8/KzznjlzRuXLl9fjjz8uR0dH+fr6qn79+vla753iFnoAAAAAKAGqVq2qRo0aWa9snzx5Urt27bIObnfjxg199NFHatWqlWrVqqWgoCBt3bpVZ86cydfyY2Ji5Ovray3eJalhw4bZ+q1Zs0Zdu3ZVvXr1FBQUpH/961/5Xsdf11WjRg1r8S5JjRs3VmZmpo4fP25tCw4Olr29vfXzww8/rPj4+Hyt48qVK4qLi1Pjxo1t2hs1aqSYmBhJUu/evXXw4EE9/vjjGjdunH766Sdrv86dOys1NVWPPfaYXn/9dX3//ffKyMgoUJ4FxRV4AAAAAMiD4eyss/+/oCuKdRdE3759NXbsWE2ePFmRkZHy8/PT448/Lunm1eY5c+bovffeU1BQkFxcXDRhwoR8PzNuGEa2Nsstt/fv3btXw4YN06uvvqrWrVurbNmyWrlypWbPnl2gPAzDyLbsnNbp6OiYbVpOcebl1vX8dd21a9fWzz//rM2bN2v79u0aOnSoWrRooTlz5sjX11dbt27Vtm3btG3bNr311luaOXOmli1bli2uwkIBDwAAAAB5sVjyfRt7UXvyySc1fvx4LV++XN9++6369+9vLUZ37dql9u3bq1evXkpPT1dmZqZOnDihoHw+HhAcHKwzZ84oLi5OPj4+km4W7H/1yy+/yM/PTyNHjrS23Xr13dHRUZmZmbdd19KlS5WcnGy9Cv/LL7/Izs5OAQEB+Yr3dsqWLSsfHx/t3r1bzZo1s7bv3btX9erVs+n31FNP6amnnlKnTp3Uv39/Xbp0SZ6ennJ2dlZYWJjCwsI0cOBAtWrVSr/99tsdjfifHxTwAAAAAFBClClTRl26dFFERISSkpLUu3dv67QqVapo3bp12r17t1xdXTV79mxduHAh3wX8448/rsDAQI0cOVLjx4/X1atXFRERYdPH399fZ86c0cqVK1W3bl1t2rRJ33//vU2fSpUq6Y8//tCvv/6qihUrqkyZMtleH9e9e3d9+OGHGjlypF599VVdvHhR48aNU48ePazPvxeGoUOH6sMPP9QjjzyiWrVqacmSJTp48KB1pPnZs2fr4YcfVq1atWSxWLRmzRp5e3vL3d1dkZGRyszMVP369eXs7Kxly5bJyclJvr6+hRbfrXgGHgAAAABKkD59+igxMVGPP/64TTE5atQo1a5dW08//bR69uyp8uXLq3379vlerp2dnebOnau0tDR17txZr732mt544w2bPu3bt9dzzz2nsWPHKiwsTHv27LEZ1E6SOnbsqNatW6t3796qXbt2jq+yc3Z21jfffKPExER16tRJzz//vFq0aKFJkyYVaFvczrPPPqsXXnhB77zzjp544glt2bJF8+bNs17lL1OmjD799FN16NBBnTp10qlTp/T111/Lzs5O7u7u+uabb9S1a1fr4Hnz589XuXLlCjXGv7IYBX1A4AFw4cKFAr07sDiwWCyqUKGCzp49W+BnPoo7cjMncjMncit+HB0dC/VKA/6P2fb3Zv0dzg9yM6eSnltKSoqcnJyKOpR7ImsU+pKouOeWlJQkNzc3m7aC7Ou5Ag8AAAAAgAlQwAMAAAAAYAIMYgcAAO6ZDRs2aNWqVUpMTJSfn5/Cw8NVo0aNXPsfOnRICxYs0OnTp+Xp6akuXbooLCwsx747duzQRx99pEaNGmn06NH3KgUAAIoNrsADAIB7YufOnZo/f766d++uiIgI1ahRQ5MnT1Z8fHyO/c+fP68pU6aoRo0aioiIULdu3TRv3jz9/PPP2fpeuHBBX3/9dZ4nAwAAKGko4AEAwD2xZs0atW3bVqGhodar715eXoqKisqxf1RUlLy8vBQeHi4/Pz+FhoaqTZs2Wr16tU2/zMxMffzxx+rdu7e8vb3vRyoAABQLFPAAAKDQZWRkKDY2VnXr1rVpr1Onjo4cOZLjPDExMapTp45NW7169RQbG6uMjAxr29KlS+Xm5qa2bdsWfuAA8P9ZLJaiDgEl0N2+sYFn4AEAQKFLSkpSZmam3N3dbdrd3d2VmJiY4zyJiYk59r9x44auXLkiT09P/fbbb9q8ebPef//9fMeSnp5u80ohi8UiZ2dn689mkRWrmWLOL3Izp5Kem6Ojo9LS0lSqVKmiDgclgGEYSk5OloODw139n6GABwAA90xOByl5HbjcOi3rSkXWO5lnzJihF154Ids7dPOyfPlyLV261PrZ399fERER+X7nbnHj4+NT1CHcM+RmTiU1t8zMTP3555+6fv16iXvPfWpqalGHcM8U19wsFovKly8vDw+Pu1oOBTwAACh0bm5usrOzy3a1/fLly9musmfx8PDI1j8pKUn29vZydXXV6dOndeHCBUVERFinZx1U9+nTR9OnT8+xkOjWrZs6d+5s/Zx1kuDChQs2t+YXdxaLRT4+PoqLiytxxQS5mdODkJuDg4Ps7e2LOpxC9SB8b8U1t5SUFKWkpGRrd3BwyPdJZQp4AABQ6BwcHBQQEKDo6Gg1adLE2h4dHa3GjRvnOE9QUJD27t1r03bgwAEFBATIwcFBFStW1AcffGAzffHixUpNTbUOkJcTR0dHOTo65jitOB7g3Y5hGKaMOz/IzZzIzZzIzZwYxA4AANwTnTt31qZNm7R582adPn1a8+fPV3x8vNq1aydJWrhwoT755BNr/7CwMMXHx1vfA79582Zt3rxZTz75pCSpVKlSqly5ss2/MmXKyMnJSZUrV5aDA9clAAAlG3s6AABwTzRv3lxXrlzRsmXLdOnSJVWqVEljxoyx3iZ46dIlm3fCe3t7a8yYMVqwYIE2bNggT09PDRo0SM2aNSuqFAAAKFYo4AEAwD3Tvn17tW/fPsdpw4cPz9ZWs2ZNm2fcbyenZQAAUFJxCz0AAAAAACZAAQ8AAAAAgAlQwAMAAAAAYAIU8AAAAAAAmAAFPAAAAAAAJkABDwAAAACACVDAAwAAAABgAhTwAAAAAACYAAU8AAAAAAAmQAEPAAAAAIAJUMADAAAAAGACFPAAAAAAAJgABTwAAAAAACZAAQ8AAAAAgAlQwAMAAAAAYAIU8AAAAAAAmAAFPAAAAAAAJkABDwAAAACACVDAAwAAAABgAhTwAAAAAACYAAU8AAAAAAAmQAEPAAAAAIAJUMADAAAAAGACFPAAAAAAAJgABTwAAAAAACZAAQ8AAAAAgAlQwAMAAAAAYAIU8AAAAAAAmAAFPAAAAAAAJkABDwAAAACACVDAAwAAAABgAhTwAAAAAACYAAU8AAAAAAAmQAEPAAAAAIAJUMADAAAAAGACFPAAAAAAAJgABTwAAAAAACZAAQ8AAAAAgAlQwAMAAAAAYAIU8AAAAAAAmAAFPAAAAAAAJkABDwAAAACACVDAAwAAAABgAg53MtOGDRu0atUqJSYmys/PT+Hh4apRo0au/Q8dOqQFCxbo9OnT8vT0VJcuXRQWFmbTZ+3atYqKilJ8fLzc3NzUtGlT9evXT6VKlZIkpaSkKDIyUrt379bly5fl7++v8PBwVa1a1WY5p0+f1jfffKNDhw7JMAxVqlRJr7zyiry8vO4kVQAAAAAAioUCF/A7d+7U/PnzNWTIEFWrVk0bN27U5MmTNW3atByL5PPnz2vKlCkKDQ3VSy+9pCNHjmju3Llyc3NTs2bNJEnbtm3TwoUL9eKLLyo4OFhnz57VZ599JkkKDw+XJH3++ec6deqURowYoXLlymnr1q169913NW3aNJUrV06SFBcXp/Hjx6tt27bq3bu3XFxcdObMGTk6Ot7p9gEAAAAAoFgo8C30a9asUdu2bRUaGmq9+u7l5aWoqKgc+0dFRcnLy0vh4eHy8/NTaGio2rRpo9WrV1v7HD16VNWqVVOLFi3k7e2tunXrKiQkRLGxsZKktLQ07dq1SwMGDFDNmjXl4+Oj3r17y9vb22a9ixcvVv369TVgwAD5+/vr4YcfVoMGDeTu7l7QNAEAAAAAKFYKdAU+IyNDsbGx6tq1q017nTp1dOTIkRzniYmJUZ06dWza6tWrpy1btigjI0MODg6qXr26tm3bpmPHjqlq1ao6d+6c9u3bp1atWkmSbty4oczMzGxX0kuVKqXffvtNkpSZman//Oc/6tKliyZNmqQTJ07I29tbXbt2VZMmTXKMLT09Xenp6dbPFotFzs7O1p/NJCtes8WdH+RmTuRmTuQGAABQfBWogE9KSlJmZma2K9ru7u5KTEzMcZ7ExMQc+9+4cUNXrlyRp6enQkJClJSUpHHjxkm6WbCHhYVZTxQ4OzsrODhYy5Ytk6+vrzw8PLR9+3YdO3ZMPj4+1thSU1O1cuVKPf300+rfv7/279+vDz/8UBMmTFDNmjWzxbZ8+XItXbrU+tnf318REREqX758QTZLsZK1PUoicjMncjMncgMAACh+7mgQu5yuXuR1RePWaYZh2LQfPHhQ3333nYYMGaKgoCDFxcVp3rx58vDwUM+ePSVJI0aM0MyZMzV06FDZ2dnJ399fISEhOnHihKSbV+AlqVGjRurcubMkqUqVKjpy5IiioqJyLOC7detm7fvXeC5cuKCMjIz8bYxiwmKxyMfHR3FxcdbtW1KQmzmRmzmRW/Hj4OBg6hPLAACg8BSogHdzc5OdnV22q+2XL1/O9TlzDw+PbP2TkpJkb28vV1dXSVJkZKRatmyp0NBQSVLlypWVmpqq2bNnq3v37rKzs5OPj48mTpyo1NRUpaSkyNPTU9OmTZO3t7c1Nnt7e/n5+dmsy9fXN9fb+x0dHXMd4M5MB3d/ZRiGaWO/HXIzJ3IzJ3IDAAAofgo0iJ2Dg4MCAgIUHR1t0x4dHa1q1arlOE9QUFC2/gcOHFBAQIAcHG6eP7h+/Xq2q/R2dnY5HmA5OTnJ09NTV69e1YEDB9S4cWNrbIGBgfrzzz9t+p89e5ZXyAEAAAAATK/Ao9B37txZmzZt0ubNm3X69GnNnz9f8fHxateunSRp4cKF+uSTT6z9w8LCFB8fb30P/ObNm7V582Y9+eST1j4NGzbUDz/8oB07duj8+fOKjo5WZGSkGjVqJDu7myHu379f+/fvt06fOHGiKlasqNatW1uX06VLF+3cuVMbN25UXFyc1q9fr71796p9+/Z3un0AAAAAACgWCvwMfPPmzXXlyhUtW7ZMly5dUqVKlTRmzBjr83mXLl1SfHy8tb+3t7fGjBmjBQsWaMOGDfL09NSgQYOs74CXpB49eshisWjx4sVKSEiQm5ubGjZsqL59+1r7JCcna9GiRbp48aJcXV3VtGlT9e3b13oVX5KaNGmi5557TitWrNC8efNUsWJFvfrqq6pevfodbRwAAAAAAIqLOxrErn379rle1R4+fHi2tpo1ayoiIiLX5dnb26tXr17q1atXrn2aN2+u5s2b3za2tm3bqm3btrftBwAAAACAmRT4FnoAAAAAAHD/UcADAAAAAGACFPAAAAAAAJgABTwAAAAAACZAAQ8AAAAAgAnc0Sj0AAAA+bFhwwatWrVKiYmJ8vPzU3h4uGrUqJFr/0OHDmnBggU6ffq0PD091aVLF4WFhVmnb9y4UVu3btWpU6ckSQEBAerbt6+qVq16z3MBAKCocQUeAADcEzt37tT8+fPVvXt3RUREqEaNGpo8ebLi4+Nz7H/+/HlNmTJFNWrUUEREhLp166Z58+bp559/tvY5dOiQQkJCNGHCBL333nt66KGH9N577ykhIeF+pQUAQJGhgAcAAPfEmjVr1LZtW4WGhlqvvnt5eSkqKirH/lFRUfLy8lJ4eLj8/PwUGhqqNm3aaPXq1dY+L7/8stq3b68qVarI19dXQ4cOlWEY+u9//3u/0gIAoMhQwAMAgEKXkZGh2NhY1a1b16a9Tp06OnLkSI7zxMTEqE6dOjZt9erVU2xsrDIyMnKc5/r168rIyJCrq2vhBA4AQDHGM/AAAKDQJSUlKTMzU+7u7jbt7u7uSkxMzHGexMTEHPvfuHFDV65ckaenZ7Z5vvnmG5UrV061a9fONZb09HSlp6dbP1ssFjk7O1t/NousWM0Uc36RmzmRmzmRm7lRwAMAgHsmp4OovA6sbp1mGEau86xcuVI7duzQ22+/rVKlSuW6zOXLl2vp0qXWz/7+/oqIiFD58uVvG39x5OPjU9Qh3DPkZk7kZk7kZk4U8AAAoNC5ubnJzs4u29X2y5cvZ7vKnsXDwyNb/6SkJNnb22e7RX7VqlVavny5xo0bp0ceeSTPWLp166bOnTtbP2edDLhw4UKut+YXRxaLRT4+PoqLi7Oe2CgpyM2cyM2cyK34cXBwyPdJZQp4AABQ6BwcHBQQEKDo6Gg1adLE2h4dHa3GjRvnOE9QUJD27t1r03bgwAEFBATIweH/DllWrVqlZcuWaezYsQoMDLxtLI6OjnJ0dMxxmpkO8LIYhmHKuPOD3MyJ3MyJ3MyJQewAAMA90blzZ23atEmbN2/W6dOnNX/+fMXHx6tdu3aSpIULF+qTTz6x9g8LC1N8fLz1PfCbN2/W5s2b9eSTT1r7rFy5UosXL9aLL74ob29vJSYmKjExUampqfc9PwAA7jeuwAMAgHuiefPmunLlipYtW6ZLly6pUqVKGjNmjPU2wUuXLtm8E97b21tjxozRggULtGHDBnl6emrQoEFq1qyZtU9UVJQyMjI0depUm3X17NlTvXv3vj+JAQBQRCjgAQDAPdO+fXu1b98+x2nDhw/P1lazZk1FRETkurxPP/200GIDAMBsuIUeAAAAAAAToIAHAAAAAMAEKOABAAAAADABCngAAAAAAEyAAh4AAAAAABOggAcAAAAAwAQo4AEAAAAAMAEKeAAAAAAATIACHgAAAAAAE6CABwAAAADABCjgAQAAAAAwAQp4AAAAAABMgAIeAAAAAAAToIAHAAAAAMAEKOABAAAAADABCngAAAAAAEyAAh4AAAAAABOggAcAAAAAwAQo4AEAAAAAMAEKeAAAAAAATIACHgAAAAAAE6CABwAAAADABCjgAQAAAAAwAQp4AAAAAABMgAIeAAAAAAAToIAHAAAAAMAEKOABAAAAADABCngAAAAAAEyAAh4AAAAAABOggAcAAAAAwAQo4AEAAAAAMAEKeAAAAAAATIACHgAAAAAAE6CABwAAAADABCjgAQAAAAAwAQp4AAAAAABMgAIeAAAAAAAToIAHAAAAAMAEKOABAAAAADABCngAAAAAAEyAAh4AAAAAABNwKOoAAMDsMjIylJycXNRhFJqUlBSlpaUVdRj3RHHNzcXFRQ4O7JIBAEDeOFoAgLuQkZGha9euqWzZsrKzKxk3NTk6Oio9Pb2ow7gnimNumZmZunLlisqUKUMRDwAA8lQyjjYBoIgkJyeXqOId95+dnZ3Kli1bou7iAAAA9wZHnABwlyjecbf4HQIAAPnBEQMAAAAAACZAAQ8AKBQ9e/bU+PHj893/1KlT8vX11a+//noPowIAACg5GC0HAB4wvr6+eU5/+umnNXXq1AIvd86cOXJ0dMx3/4oVK2rfvn0qV65cgdcFAADwIKKAB4AHzL59+6w/r1q1Sh988IG2bt1qbStbtqxN//T09HwV5p6engWKw97eXt7e3gWaBwAA4EHGLfQA8IDx9va2/itbtqwsFov18/Xr11W1alWtWrVKPXv2VEBAgL777jslJCRo2LBhatiwoQIDAxUaGqoVK1bYLPfWW+ibNm2qjz/+WP/4xz8UHBysxo0b69///rd1+q230O/cuVO+vr7atm2bOnTooMDAQHXp0kXHjh2zWc/06dNVp04dBQcH67XXXtPkyZPVrl27XPO9ceOGXn31VTVr1kyVK1fW448/rrlz52brt3jxYrVp00b+/v6qX7++xo4da512+fJljR49WnXr1lVAQIDatm2rH374oUDbHQAA4G5xBR4ACpFhSCkplvu+XmdnQ5ZCXO3kyZM1fvx4TZ06VaVKldL169dVp04dDRs2TGXLltWmTZv08ssvq3LlymrQoEGuy5k1a5Zef/11vfTSS1q7dq3GjBmjZs2aqWrVqrnOExERofHjx+uhhx7Sm2++qVdffVUrV66UJH333XeaMWOGJk+erEaNGmnVqlWaNWuWKlWqlOvyMjMzVaFCBX3++efy9vbWzz//rNGjR8vb21tdunSRJC1YsEDvvPOOxowZozZt2ujKlSv65ZdfrPMPGDBA165d04wZM/TII4/o6NGjsre3v5NNCwAAcMco4AGgEKWkWBQUVOG+rzcm5qxcXIxCW96QIUPUsWNHm7ahQ4dafx48eLC2bNmiNWvW5FnAt23bVuHh4ZKk4cOHa86cOdq5c2eeBfwbb7yhxx57zDrPM888o9TUVDk5OenLL79Unz599PTTT0uSXnnlFf3000+6du1arstzdHTUa6+9Zv25YsWK2rNnj1avXm0t4D/++GM9//zzGjJkiHW+evXqSZK2bdum/fv368cff1RgYKAk6ZFHHsl1fQAAAPcKBTwAIJu6devafL5x44Y++eQTrV69WmfPnlVaWprS0tLk4uKS53Jq1qxp/dlisah8+fK6ePFivud5+OGHJUkXL16Ur6+vYmNjNXDgQJv+9erV044dO/Jc5ldffaVFixbpzJkzSklJUXp6umrVqiVJio+PV1xcnFq0aJHjvAcPHlSFChWsxTsAAEBRoYAHgELk7GwoJuZskay3cJfnbPN51qxZmjNnjiZOnKjq1avLxcVFEyZMUHp6ep7LcXCw3c1YLBZlZmYWaB5JNvNYbnlWwDDyzn3VqlWaOHGixo0bp2bNmql06dKaOXOmdTA/JyenPOe/3XQAAID7hQIeAAqRxaJCvZW9uNi1a5fat2+vHj16SLpZUJ84cUJBQUH3NY6AgADt379fPXv2tLZFR0fnOc/u3bvVsGFDhYeHy9HRUenp6fr999+t011dXVWpUiVt375dISEh2eavUaOGzp49q+PHj3MVHgAAFKk7KuA3bNigVatWKTExUX5+fgoPD1eNGjVy7X/o0CEtWLBAp0+flqenp7p06aKwsDCbPmvXrlVUVJTi4+Pl5uampk2bql+/fipVqpQkKSUlRZGRkdq9e7cuX74sf39/hYeH5/oc5ezZs7Vx40YNHDhQnTp1upM0AQD/X5UqVbRu3Tr98ssv8vDw0OzZs3XhwoX7XsAPHjxYr7/+uurUqWMdxO7w4cOqXLlyrvNUqVJFS5cu1Y8//qiAgAAtXrxYBw4csBn47h//+IfGjBkjLy8vtWnTRteuXdMvv/yiwYMH67HHHlPTpk31/PPPa8KECapSpYqOHTsmi8WiNm3a3I+0AQAAJN1BAb9z507Nnz9fQ4YMUbVq1bRx40ZNnjxZ06ZNk5eXV7b+58+f15QpUxQaGqqXXnpJR44c0dy5c+Xm5qZmzZpJujlA0MKFC/Xiiy8qODhYZ8+e1WeffSZJ1sGPPv/8c506dUojRoxQuXLltHXrVr377ruaNm2aypUrZ7PO3bt3KyYmpsDvJAYA5GzUqFE6deqU+vfvL2dnZ/Xv31/t27fXlStX7msc3bt31++//653331X169f15NPPqnevXvbvNv+Vn//+9918OBBvfjii7JYLHrqqac0cOBAbd682dqnd+/eun79uubMmaN3331X5cqVszn5m9U+bNgwpaSkqEqVKhozZsw9zRUAAOBWFuN2Dw/e4q233pK/v7+ee+45a9srr7yixo0bq1+/ftn6//vf/9bevXs1bdo0a9vs2bP1+++/a9KkSZKkL774QmfOnLF5f/BXX32lY8eO6Z133lFaWpqeeeYZjR492ma049dff10NGzZUnz59rG0JCQl66623NHbsWP3zn/9Ux44dC3wF/sKFC7d9rrO4sVgsqlChgs6ePXvb50HNhtzM6UHJ7fLly3JzcyvqkApV1m3mZtGnTx+VL19eM2bMuG3f4pxbUlJSjr9Ljo6OKl++fBFEVPKZbX//oPxdJTfzIDdzIrfipyD7+gJdgc/IyFBsbKy6du1q016nTh0dOXIkx3liYmJUp04dm7Z69eppy5YtysjIkIODg6pXr65t27bp2LFjqlq1qs6dO6d9+/apVatWkm6OfpyZmSlHR0eb5ZQqVUq//fab9XNmZqZmzJihLl265PlO4Czp6ek2O26LxWIduOnWQZKKu6x4zRZ3fpCbOZEb7oWUlBR99dVXat26tezt7bVixQpt27ZNixYtKurQCgW/UwAAIC8FKuCTkpKUmZkpd3d3m3Z3d3clJibmOE9iYmKO/W/cuKErV67I09NTISEhSkpK0rhx4yTdLNjDwsKsJwqcnZ0VHBysZcuWydfXVx4eHtq+fbuOHTsmHx8f63JXrlwpe3t7dejQIV/5LF++XEuXLrV+9vf3V0REhKmvdPx1e5Q05GZOJT235OTkbCcXS4LimlNGRoa2bNmijz/+WGlpaQoMDNSXX36p0NDQfC+juOZWqlQpVahQoajDKHT3Ytycn3/+WZGRkTp37pwefvhh9e3bV02aNLnXqQAAUOTuaBC7nK4Q5HXVILdX/mS1Hzx4UN99952GDBmioKAgxcXFad68efLw8LCONDxixAjNnDlTQ4cOlZ2dnfz9/RUSEqITJ05IkmJjY7Vu3TpFRETk+wpGt27d1Llz52xxXrhwQRkZGflaRnFhsVjk4+OjuLg4U90ukh/kZk4PSm5paWmmugU3P4rzbeYODg5avHhxtvb8xlucc0tLS9PZs9lfQejg4GDaE8v3Ytyco0ePavr06Xr66afVpEkT7d69W9OmTdM777xz3wdVBADgfitQAe/m5iY7O7tsV9svX76c7Sp7Fg8Pj2z9k5KSZG9vL1dXV0lSZGSkWrZsab2CUrlyZaWmpmr27Nnq3r277Ozs5OPjo4kTJyo1NVUpKSny9PTUtGnT5O3tLUk6fPiwkpKSNGzYMOt6MjMz9dVXX2ndunX69NNPs8Xm6OiY65UYsxYchmGYNvbbITdzKum5AYWppP1OrVmzRm3btrXu38PDw3XgwAFFRUXlOG5OVFSUvLy8rAPY+vn56fjx41q9erW1gF+7dq3q1Kmjbt26Sbp5Mv7QoUNau3atRo0adV/yAgCgqBSogHdwcFBAQICio6NtblWLjo5W48aNc5wnKChIe/futWk7cOCAAgIC5OBwc/XXr1/PdtXczs4uxwMZJycnOTk56erVqzpw4IAGDBggSWrZsqVq165t03fSpElq2bIlr/kBAOA+u1fj5hw9ejTb4LR169bVunXrCjX+/DAyDaUmpN639VksFl2zv6aU+JQSd7KH3MyJ3MyJ3AqXUzknWezu3xg2Bb6FvnPnzpoxY4YCAgIUHBysjRs3Kj4+Xu3atZMkLVy4UAkJCRoxYoQkKSwsTBs2bNCCBQsUGhqqo0ePavPmzRo5cqR1mQ0bNtTatWvl7+9vvYU+MjJSjRo1kp2dnSRp//79kqSKFSsqLi5OX3/9tSpWrKjWrVtLksqWLauyZcvaJufgIA8PD1WsWLHAGwYAANy5ezVuTmJiojw8PGz65HS331/dq0FrUy+lKrBu1Tue/04F3Pc13j/kZk7kZk7kVjhio4/L2cv5vq2vwAV88+bNdeXKFS1btkyXLl1SpUqVNGbMGOvzeZcuXVJ8fLy1v7e3t8aMGaMFCxZow4YN8vT01KBBg6y3wklSjx49ZLFYtHjxYiUkJMjNzU0NGzZU3759rX2Sk5O1aNEiXbx4Ua6urmratKn69u1rvYoPAACKn8IeNycnhmHkOf1eDVp7zf7aXc0PADC/hx9+WGW8y9y39d1R9du+fXu1b98+x2nDhw/P1lazZk1FRETkujx7e3v16tVLvXr1yrVP8+bN1bx58wLFmdNz7wAA4N67V+Pm5NQnr2VK927QWiPT0Lno43c8f0FZLJK398M6f/6cSthdr+RmUuRmTuRWuJzSLyvpbNJdLaMgA9Zy+RoAABS6ezVuTnBwsP773//aFOTR0dEKDg7ONZZ7NmitRXJ6yOnO5y/o6iwWlfEuI6cbziXyuVVyMx9yMydyK3z3c112921NAIASpWfPnho/frz1c9OmTTVnzpw85/H19dX69evvet2FtRzcW507d9amTZu0efNmnT59WvPnz882bs4nn3xi7R8WFqb4+Hjre+A3b96szZs368knn7T26dixow4cOKAVK1bozJkzWrFihf773/9mG9gOAICSiCvwAPCAGThwoFJTUxUZGZlt2p49e/TUU09p/fr12d7scTvr1q2Ti4tLYYUpSfrwww+1fv16/fDDDzbt+/bty/OWaRQP92LcnGrVqmnUqFFavHixIiMj5ePjo1GjRvEOeADAA4ECHgAeMH379tWQIUN0+vRp+fn52UyLjIzUo48+WuDiXZIeeuihwgrxtry9ve/bunB3CnvcHElq1qyZTVEPAMCDglvoAeAB88QTT8jLy0tLliyxaU9JSdGqVavUv39/JSQkaNiwYWrYsKECAwMVGhqqFStW5LncW2+hj42NVffu3RUQEKDWrVtr69at2eaZNGmSWrRoocDAQD322GN6//33ra/7ioyM1NSpU3Xo0CH5+vrK19fXetfArbfQHz58WL169VJgYKBq1aql0aNH69q1/xshfNSoURo8eLA+/fRT1a9fX7Vq1dJbb71l82qxW508eVKDBg1S3bp1FRQUpI4dO2bL4fr163rvvffUqFEj+fv7KyQkRIsWLbJOP3LkiP7+97+rWrVqCg4OVrdu3XTy5Mk8tyMAAEBuuAIPAIXJMGRJSbn/q3V2vjn0aj44ODioZ8+eWrJkiV555RXriNyrV69Wenq6evTooaSkJNWpU0fDhg1T2bJltWnTJr388suqXLmyGjRocNt1ZGZm6rnnnpOnp6dWrVqlq1evasKECdn6lSlTRtOmTZOPj48OHz6s0aNHy9XVVcOGDVOXLl105MgR/fjjj1q8eLEkqWzZstmWkZKSogEDBqhBgwZau3at4uPj9frrr2vs2LGaPn26td/OnTvl4+Ojb7/9VidOnNCLL76oWrVqqX///jnmcO3aNbVt21ajR49W6dKl9e2332rQoEHaunWrfH19JUkjR47U3r179e6776pmzZr6448/lJCQIEk6e/asunfvrubNm2vJkiVydXXVnj177mrUcwAA8GCjgAeAQmRJSVGFIngW92xMjIwCPH/ep08fzZw5Uzt37lRISIikm1e8O3ToIA8PD5UpU0ZDhw619h88eLC2bNmiNWvW5KuA37Ztm2JiYvTzzz+rYsWKkqQ333xTAwYMsOk3atQo68+VKlXS8ePHtWrVKg0bNkzOzs4qU6aM7O3t87xl/rvvvlNqaqo++ugj6zP47733nsLDwzV27Fjr89bu7u765z//qczMTFWtWlWhoaHavn17rgV8rVq1VKtWLevnN954Q+vXr1dUVJQGDRqk48ePa/Xq1Vq0aJFatmwpSXrkkUes/efPny83Nzd99tln1hHQAwMDb7vtAAAAckMBDwAPoKpVq6pRo0ZavHixQkJCdPLkSe3atUsLFy6UJN24cUOffPKJVq9erbNnzyotLU1paWn5HqQuJiZGvr6+1uJdkho2bJit35o1azR37lydPHlS165d040bN6zv+86vmJgY1ahRwya2xo0bKzMzU8ePH7cW8MHBwbK3t1dmZqYk6eGHH9bhw4dzXW5ycrKmTp2qjRs36ty5c8rIyFBqaqrOnDkjSTp48KDs7e312GOP5Tj/oUOH1KRJk1xfXwYAAFBQFPAAUIgMZ2edjYkpkvUWVN++fTV27FhNnjxZkZGR8vPz0+OPPy5JmjVrlubMmaOJEyeqevXqcnFx0YQJE/J8Ztwmnhzeh2q55Rb/vXv3atiwYXr11VfVunVrlS1bVitXrtTs2bMLlIdhGNmWndM6cyqk83pv67vvvquffvpJ48aNU5UqVeTk5KTnn39eaWlpkiQnp7zf/3276QAAAAVFAQ8AhcliKdCt7EXpySef1Pjx47V8+XJ9++236t+/v7Xg3bVrl9q3b68ePXpIuvlM+4kTJ/L9qq7g4GCdOXNGcXFx8vHxkXSzYP+rX375RX5+fho5cqS1LevqdhZHR0frFfO81rV06VIlJydbr8L/8ssvsrOzU0BAQL7izcnu3bvVq1cvdejQQdLNZ+JPnz5tnV6jRg1lZmbqf//3f6230P9VjRo19O233yo9PZ2r8AAAoFAwCj0APKDKlCmjLl26KCIiQufOnVPv3r2t06pUqaKtW7fql19+UUxMjN544w1duHAh38t+/PHHFRgYqJEjR+rgwYPatWtXtleD+fv768yZM1q5cqVOnjypL774Qt9//71Nn0qVKumPP/7Qr7/+qoSEBF2/fj3burp3767SpUtr5MiR+u2337Rjxw6NGzdOPXr0sN4+fyeqVKmi77//Xr/++qsOHjyo4cOH25xMqFSpknr16qVXX31V69ev1x9//KGdO3dq1apVkqTw8HBduXJFw4YN04EDBxQbG6ulS5fq2LFjdxwTAAB4sFHAA8ADrE+fPkpMTNTjjz9uHVldujm4XO3atdW/f3/17NlT5cuXz/Vd3jmxs7PT3LlzlZaWps6dO+u1117TG2+8YdOnffv2eu655zR27FiFhYVpz549NoPaSVLHjh3VunVr9e7dW7Vr187xVXbOzs765ptvlJiYqE6dOun5559XixYtNGnSpAJti1u9/fbbcnd311NPPaXw8HC1bt1atWvXtukzZcoUderUSW+99ZZatWql119/XSn//y0E5cqV05IlS3Tt2jX16NFDHTp00MKFC7kaDwAA7pjFyOsBwAfUhQsX8v2cZ3FhsVhUoUIFnT17Ns9nOs2I3MzpQcnt8uXLcnNzK+qQCpWjo6Pp/gbmV3HOLSkpKcffJUdHx7u6kwC5M9v+/kH5u0pu5kFu5kRuxU9B9vVcgQcAAAAAwAQo4AEAAAAAMAEKeAAAAAAATIACHgAAAAAAE6CABwAAAADABCjgAQAAAAAwAQp4ALhLmZmZRR0CTI7fIQAAkB8U8ABwF1xcXHTlyhUKMNyxzMxMXblyRS4uLkUdCgAAKOYcijoAADAzBwcHlSlTRlevXi3qUApNqVKllJaWVtRh3BPFNbcyZcrIwYFdMgAAyBtHCwBwlxwcHOTm5lbUYRQKi8WiChUq6OzZszIMo6jDKVQlOTcAAPBg4BZ6AAAAAABMgAIeAAAAAAAToIAHAAAAAMAEKOABAAAAADABCngAAAAAAEyAAh4AAAAAABOggAcAAAAAwAQo4AEAAAAAMAEKeAAAAAAATIACHgAAAAAAE6CABwAAAADABCjgAQAAAAAwAQp4AAAAAABMgAIeAAAAAAAToIAHAAAAAMAEKOABAAAAADABCngAAAAAAEyAAh4AAAAAABOggAcAAAAAwAQo4AEAAAAAMAEKeAAAAAAATIACHgAAAAAAE6CABwAAAADABCjgAQAAAAAwAQp4AAAAAABMgAIeAAAAAAAToIAHAAAAAMAEKOABAAAAADABCngAAAAAAEyAAh4AAAAAABOggAcAAAAAwAQo4AEAAAAAMAEKeAAAAAAATIACHgAAAAAAE6CABwAAAADABCjgAQAAAAAwAQp4AAAAAABMgAIeAAAAAAAToIAHAAAAAMAEKOABAAAAADABh6IOAAAAlDxXr17VvHnztGfPHklSo0aNNHjwYJUpUybXeQzD0LfffqtNmzbp6tWrCgoK0rPPPqtKlSpZl7lkyRIdOHBAFy9eVNmyZdW4cWP16dNHLi4u9yUvAACKElfgAQBAofv444918uRJjR07VmPHjtXJkyc1Y8aMPOdZuXKl1q5dq8GDB2vKlCny8PDQe++9p5SUFElSQkKCEhIS9Pe//10ffPCBhg8frgMHDmjmzJn3IyUAAIocBTwAAChUp0+f1v79+zV06FAFBwcrODhYL7zwgv7zn//ozz//zHEewzC0bt06devWTU2bNlXlypU1fPhwXb9+Xdu3b5ckVa5cWa+99poaNWokHx8fPfroo+rTp4/27t2rGzdu3M8UAQAoEhTwAACgUB09elQuLi4KCgqytgUHB8vFxUVHjhzJcZ7z588rMTFRdevWtbY5OjqqZs2auc4jScnJyXJ2dpa9vX3hJQAAQDHFM/AAAKBQJSYmyt3dPVu7u7u7EhMTc50nq8+t88THx+c4z5UrV7Rs2TK1a9cuz3jS09OVnp5u/WyxWOTs7Gz92SyyYjVTzPlFbuZEbuZEbuZGAQ8AAPJlyZIlWrp0aZ59pkyZkus0wzBue1B163TDMHLsl5ycrH/+85/y8/NTz54981zm8uXLbeL29/dXRESEypcvn+d8xZWPj09Rh3DPkJs5kZs5kZs5UcADAIB8+dvf/qaQkJA8+5QvX16///67Ll++nG1aUlJSjlfmJcnDw0PSzSvxnp6eec6TkpKiyZMny8nJSa+99pocHPI+nOnWrZs6d+5s/Zx1kuDChQvKyMjIc97ixGKxyMfHR3Fxcbme2DArcjMncjMncit+HBwc8n1SmQIeAADki5ubm9zc3G7bLzg4WMnJyTp27JiqVq0qSYqJiVFycrKqVauW4zze3t7y8PBQdHS0/P39JUkZGRk6dOiQ+vfvb+2XnJysSZMmydHRUaNHj1apUqVuG4+jo6McHR1znGamA7wshmGYMu78IDdzIjdzIjdzuqMCfsOGDVq1apUSExPl5+en8PBw1ahRI9f+hw4d0oIFC3T69Gl5enqqS5cuCgsLs+mzdu1aRUVFKT4+Xm5ubmratKn69etn3TGnpKQoMjJSu3fv1uXLl+Xv76/w8HDrgUFGRoYWL16sffv26fz583JxcVHt2rXVr18/lStX7k7SBAAAd8DPz0/16tXTrFmz9Nxzz0mSZs+erQYNGqhixYrWfqNGjVK/fv3UpEkTWSwWdezYUcuXL1eFChXk4+Oj5cuXq3Tp0mrRooWkm8cCkyZN0vXr1/XSSy8pJSXF+oo5Nzc32dkxNi8AoGQrcAG/c+dOzZ8/X0OGDFG1atW0ceNGTZ48WdOmTZOXl1e2/ufPn9eUKVMUGhqql156SUeOHNHcuXPl5uamZs2aSZK2bdumhQsX6sUXX1RwcLDOnj2rzz77TJIUHh4uSfr888916tQpjRgxQuXKldPWrVv17rvvatq0aSpXrpzS0tJ04sQJ9ejRQ1WqVNHVq1e1YMECvf/++/rnP/95F5sIAAAU1Msvv6wvv/xSkyZNkiQ1bNhQzz77rE2fP//8U8nJydbPTz31lNLS0jR37lxdu3ZNVatW1dixY60DzsXGxiomJsa6/L/65JNP5O3tfS9TAgCgyBW4gF+zZo3atm2r0NBQSTcL7AMHDigqKkr9+vXL1j8qKkpeXl7WQtzPz0/Hjx/X6tWrrQX80aNHVa1aNesZdm9vb4WEhOjYsWOSpLS0NO3atUujR49WzZo1JUm9e/fWL7/8oqioKPXp00cuLi4aN26czboHDRqkt956S/Hx8TmeXAAAAPeGq6trtiL7VkuWLLH5bLFY1Lt3b/Xu3TvH/rVq1co2DwAAD5ICFfAZGRmKjY1V165dbdrr1KmT6ztaY2JiVKdOHZu2evXqacuWLcrIyJCDg4OqV6+ubdu2WZ+VO3funPbt26dWrVpJkm7cuKHMzMxsz6+VKlVKv/32W67xJicny2KxyMXFJcfpJeW1MlLJfmUCuZkTuZkTuQEAABRfBSrgk5KSlJmZmeM7WvN6r2tO/W/cuKErV67I09NTISEhSkpKsl5Bv3HjhsLCwqwnCpydnRUcHKxly5bJ19dXHh4e2r59u44dO5brKwLS0tK0cOFChYSE5FrAl7TXykgl+5UJ5GZO5GZO5AYAAFD83NEgdjldvcjrikZu73TNaj948KC+++47DRkyREFBQYqLi9O8efPk4eFhfbfriBEjNHPmTA0dOlR2dnby9/dXSEiITpw4kW19GRkZmj59ugzD0JAhQ3KNq6S8VkYy7ysT8oPczInczIncip+CvFoGAACUbAUq4LNGeL31avvly5fzfK/rrf2TkpJkb28vV1dXSVJkZKRatmxpfa6+cuXKSk1N1ezZs9W9e3fZ2dnJx8dHEydOVGpqqlJSUuTp6alp06ZlG7AmIyND06ZN04ULFzR+/Phcr75LJe+1MlLJfmUCuZkTuZkTuQEAABQ/BXrfioODgwICAhQdHW3THh0dnet7XYOCgrL1P3DggAICAuTgcPP8wfXr17Ndpbezs8vxAMvJyUmenp66evWqDhw4oMaNG1unZRXvcXFxGjdunMqWLVuQ9AAAAAAAKLYKfAt9586dNWPGDAUEBCg4OFgbN25UfHy82rVrJ0lauHChEhISNGLECElSWFiYNmzYoAULFig0NFRHjx7V5s2bNXLkSOsyGzZsqLVr18rf3996C31kZKQaNWpkfafr/v37JUkVK1ZUXFycvv76a1WsWFGtW7eWdPO5+alTp+rEiRN64403lJmZab3y7+rqaj1ZAAAAAACAGRW4qm3evLmuXLmiZcuW6dKlS6pUqZLGjBljfT7v0qVLio+Pt/b39vbWmDFjtGDBAm3YsEGenp4aNGiQ9RVyktSjRw9ZLBYtXrxYCQkJcnNzU8OGDdW3b19rn+TkZC1atEgXL16Uq6urmjZtqr59+1oL84sXL2rPnj2SpNGjR9vEPGHCBNWqVaugqQIAAAAAUGzc0WXp9u3bq3379jlOGz58eLa2mjVrKiIiItfl2dvbq1evXurVq1eufZo3b67mzZvnOt3b25t3wwIAAAAASqwCPQMPAAAAAACKBgU8AAAAAAAmQAEPAAAAAIAJUMADAAAAAGACFPAAAAAAAJgABTwAAAAAACZAAQ8AAAAAgAlQwAMAAAAAYAIU8AAAAAAAmAAFPAAAAAAAJkABDwAAAACACVDAAwAAAABgAhTwAAAAAACYAAU8AAAAAAAmQAEPAAAAAIAJUMADAAAAAGACFPAAAAAAAJgABTwAAAAAACZAAQ8AAAAAgAlQwAMAAAAAYAIU8AAAAAAAmAAFPAAAAAAAJkABDwAAAACACVDAAwAAAABgAhTwAAAAAACYAAU8AAAAAAAmQAEPAAAAAIAJUMADAAAAAGACFPAAAAAAAJgABTwAAAAAACZAAQ8AAAAAgAlQwAMAAAAAYAIU8AAAAAAAmAAFPAAAAAAAJkABDwAAAACACVDAAwAAAABgAhTwAAAAAACYAAU8AAAAAAAmQAEPAAAAAIAJUMADAAAAAGACFPAAAAAAAJgABTwAAAAAACZAAQ8AAAAAgAlQwAMAAAAAYAIU8AAAAAAAmAAFPAAAAAAAJkABDwAAAACACVDAAwAAAABgAhTwAAAAAACYAAU8AAAAAAAmQAEPAAAAAIAJOBR1AAAAoOS5evWq5s2bpz179kiSGjVqpMGDB6tMmTK5zmMYhr799ltt2rRJV69eVVBQkJ599llVqlQpx75TpkzR/v379dprr6lJkyb3LBcAAIoLrsADAIBC9/HHH+vkyZMaO3asxo4dq5MnT2rGjBl5zrNy5UqtXbtWgwcP1pQpU+Th4aH33ntPKSkp2fquXbtWFovlXoUPAECxRAEPAAAK1enTp7V//34NHTpUwcHBCg4O1gsvvKD//Oc/+vPPP3OcxzAMrVu3Tt26dVPTpk1VuXJlDR8+XNevX9f27dtt+p48eVJr167Viy++eD/SAQCg2KCABwAAhero0aNycXFRUFCQtS04OFguLi46cuRIjvOcP39eiYmJqlu3rrXN0dFRNWvWtJnn+vXr+uijjzR48GB5eHjcsxwAACiOeAYeAAAUqsTERLm7u2drd3d3V2JiYq7zZPW5dZ74+Hjr5wULFqhatWpq3LhxvuNJT09Xenq69bPFYpGzs7P1Z7PIitVMMecXuZkTuZkTuZkbBTwAAMiXJUuWaOnSpXn2mTJlSq7TDMO47UHVrdMNw7D+vGfPHv366696//338xHt/1m+fLlN3P7+/oqIiFD58uULtJziwsfHp6hDuGfIzZzIzZzIzZwo4AEAQL787W9/U0hISJ59ypcvr99//12XL1/ONi0pKSnHK/OSrLfDJyYmytPTM8d5fv31V507d07h4eE283744YeqUaOG3n777RyX3a1bN3Xu3Nn6OeskwYULF5SRkZFnPsWJxWKRj4+P4uLibE5slATkZk7kZk7kVvw4ODjk+6QyBTwAAMgXNzc3ubm53bZfcHCwkpOTdezYMVWtWlWSFBMTo+TkZFWrVi3Heby9veXh4aHo6Gj5+/tLkjIyMnTo0CH1799fktS1a1e1bdvWZr7XXntNAwcOVKNGjXKNx9HRUY6OjjlOM9MBXhbDMEwZd36QmzmRmzmRmzlRwAMAgELl5+enevXqadasWXruueckSbNnz1aDBg1UsWJFa79Ro0apX79+atKkiSwWizp27Kjly5erQoUK8vHx0fLly1W6dGm1aNFC0s2r9DkNXOfl5SVvb+/7khsAAEWJAh4AABS6l19+WV9++aUmTZokSWrYsKGeffZZmz5//vmnkpOTrZ+feuoppaWlae7cubp27ZqqVq2qsWPHWgecAwDgQUcBDwAACp2rq6tefvnlPPssWbLE5rPFYlHv3r3Vu3fvfK/n1mUAAFCS8R54AAAAAABMgAIeAAAAAAAToIAHAAAAAMAEKOABAAAAADABCngAAAAAAEyAAh4AAAAAABOggAcAAAAAwAQo4AEAAAAAMAEKeAAAAAAATMDhTmbasGGDVq1apcTERPn5+Sk8PFw1atTItf+hQ4e0YMECnT59Wp6enurSpYvCwsJs+qxdu1ZRUVGKj4+Xm5ubmjZtqn79+qlUqVKSpJSUFEVGRmr37t26fPmy/P39FR4erqpVq1qXYRiGvv32W23atElXr15VUFCQnn32WVWqVOlO0gQAAAAAoNgo8BX4nTt3av78+erevbsiIiJUo0YNTZ48WfHx8Tn2P3/+vKZMmaIaNWooIiJC3bp107x58/Tzzz9b+2zbtk0LFy5Ur169NG3aNA0dOlT/+7//q4ULF1r7fP7554qOjtaIESP04Ycfqk6dOnr33XeVkJBg7bNy5UqtXbtWgwcP1pQpU+Th4aH33ntPKSkpBU0TAAAAAIBipcAF/Jo1a9S2bVuFhoZar757eXkpKioqx/5RUVHy8vJSeHi4/Pz8FBoaqjZt2mj16tXWPkePHlW1atXUokULeXt7q27dugoJCVFsbKwkKS0tTbt27dKAAQNUs2ZN+fj4qHfv3vL29rau1zAMrVu3Tt26dVPTpk1VuXJlDR8+XNevX9f27dvvZNsAAAAAAFBsFOgW+oyMDMXGxqpr16427XXq1NGRI0dynCcmJkZ16tSxaatXr562bNmijIwMOTg4qHr16tq2bZuOHTumqlWr6ty5c9q3b59atWolSbpx44YyMzPl6Ohos5xSpUrpt99+k3TzSn9iYqLq1q1rne7o6KiaNWvqyJEjateuXbbY0tPTlZ6ebv1ssVjk7Oxs/dlMsuI1W9z5QW7mRG7mRG4AAADFV4EK+KSkJGVmZsrd3d2m3d3dXYmJiTnOk5iYmGP/Gzdu6MqVK/L09FRISIiSkpI0btw4STcL9rCwMOuJAmdnZwUHB2vZsmXy9fWVh4eHtm/frmPHjsnHx8e6nqxl37qu3G7vX758uZYuXWr97O/vr4iICJUvXz5f26M4ytoeJRG5mRO5mRO5AQAAFD93NIhdTlcv8rqices0wzBs2g8ePKjvvvtOQ4YMUVBQkOLi4jRv3jx5eHioZ8+ekqQRI0Zo5syZGjp0qOzs7OTv76+QkBCdOHEiX+vKSbdu3dS5c+ds8164cEEZGRm5zlccWSwW+fj4KC4uLs+czYjczInczIncih8HBwdTn1gGAACFp0AFvJubm+zs7LJdbb98+XK2K99ZPDw8svVPSkqSvb29XF1dJUmRkZFq2bKlQkNDJUmVK1dWamqqZs+ere7du8vOzk4+Pj6aOHGiUlNTlZKSIk9PT02bNk3e3t7W9Ug3r8R7enrarCu32BwdHbPdlp/FTAd3f2UYhmljvx1yMydyMydyAwAAKH4KNIidg4ODAgICFB0dbdMeHR2tatWq5ThPUFBQtv4HDhxQQECAHBxunj+4fv16tivndnZ2OR5gOTk5ydPTU1evXtWBAwfUuHFjSZK3t7c8PDxs1pWRkaFDhw7lGhsAAAAAAGZR4FvoO3furBkzZiggIEDBwcHauHGj4uPjrYPELVy4UAkJCRoxYoQkKSwsTBs2bNCCBQsUGhqqo0ePavPmzRo5cqR1mQ0bNtTatWvl7+9vvYU+MjJSjRo1kp3dzXMM+/fvlyRVrFhRcXFx+vrrr1WxYkW1bt1a0s1bIzt27Kjly5erQoUK8vHx0fLly1W6dGm1aNHibrYRAAAAAABFrsAFfPPmzXXlyhUtW7ZMly5dUqVKlTRmzBjr83mXLl2yGTTO29tbY8aM0YIFC7RhwwZ5enpq0KBBatasmbVPjx49ZLFYtHjxYiUkJMjNzU0NGzZU3759rX2Sk5O1aNEiXbx4Ua6urmratKn69u1rvYovSU899ZTS0tI0d+5cXbt2TVWrVtXYsWOtI8sDAAAAAGBWFoMHAbO5cOGCzevlzMBisahChQo6e/ZsiXu2k9zMidzMidyKH0dHRwaxu0fMtr836+9wfpCbOZGbOZFb8VOQfX2BnoEHAAAAAABFgwIeAAAAAAAToIAHAAAAAMAEKOABAAAAADABCngAAAAAAEyAAh4AAAAAABOggAcAAAAAwAQcijqAksowpJQUy31bn8Vi0bVrUnKyRSZ65WG+kJs5kZs5kVvhcnY2ZLl/uwIAAFDCUcDfIykpFgUFVSiCNfsUwTrvF3IzJ3IzJ3IrDDExZ+XiUsLOhAAAgCLDLfQAAAAAAJgAV+DvEWdnQzExZ+/b+iwWi3x8fBQXFyejhN33Sm7mRG7mRG6Fy9m5ZG1DAABQtCjg7xGLRff1tkmLRSpT5uY6S95BN7mZEbmZE7kBAAAUX9xCDwAAAACACVDAAwAAAABgAhTwAAAAAACYAAU8AAAAAAAmQAEPAAAAAIAJUMADAAAAAGACFPAAAAAAAJgABTwAAAAAACZAAQ8AAAAAgAlQwAMAAAAAYAIU8AAAAAAAmAAFPAAAAAAAJkABDwAAAACACTgUdQDFkYODeTeLmWO/HXIzJ3IzJ3IrPswWr5mYdduaNe78IDdzIjdzIrfioyDxWgzDMO5hLAAAAAAAoBBwC30JkZKSojfeeEMpKSlFHUqhIzdzIjdzIjeg+CrJv8PkZk7kZk7kZm4U8CWEYRg6ceKESuINFeRmTuRmTuQGFF8l+XeY3MyJ3MyJ3MyNAh4AAAAAABOggAcAAAAAwAQo4EsIR0dH9ezZU46OjkUdSqEjN3MiN3MiN6D4Ksm/w+RmTuRmTuRmboxCDwAAAACACXAFHgAAAAAAE6CABwAAAADABCjgAQAAAAAwAQp4AAAAAABMwKGoA8DdWb58uXbv3q0zZ86oVKlSCg4O1oABA1SxYsWiDq1QLV++XIsWLVLHjh0VHh5e1OHctYSEBP373//W/v37lZaWpgoVKujFF19UQEBAUYd2V27cuKFvv/1W27ZtU2Jiojw9PdW6dWt1795ddnbmOl946NAhrVq1SidOnNClS5f02muvqUmTJtbphmHo22+/1aZNm3T16lUFBQXp2WefVaVKlYow6vzJK7eMjAwtXrxY+/bt0/nz5+Xi4qLatWurX79+KleuXBFHfnu3+97+avbs2dq4caMGDhyoTp063edIgfx7UPb1Evt7s2B/z/6+KD3o+3pz/Q9DNocOHVL79u01adIk/c///I8yMzP13nvvKTU1tahDKzTHjh3Txo0b9cgjjxR1KIXi6tWrGjdunBwcHPTWW29p6tSpeuaZZ+Ti4lLUod21lStX6ocfftCzzz6radOmacCAAVq1apXWr19f1KEV2PXr11WlShUNHjw4x+krV67U2rVrNXjwYE2ZMkUeHh567733lJKScp8jLbi8cktLS9OJEyfUo0cPRURE6NVXX9XZs2f1/vvvF0GkBXe77y3L7t27FRMTI09Pz/sUGXDnHoR9vcT+3kzY37O/L0oP+r6eK/AmN3bsWJvPw4YN05AhQxQbG6uaNWsWUVSFJzU1VTNmzNALL7yg7777rqjDKRQrV67UQw89pGHDhlnbvL29izCiwnP06FE1atRIDRo0kHQzr+3bt+v48eNFHFnB1a9fX/Xr189xmmEYWrdunbp166amTZtKkoYPH67nnntO27dvV7t27e5nqAWWV24uLi4aN26cTdugQYP01ltvKT4+Xl5eXvcjxDuWV25ZEhIS9OWXX2rs2LH65z//eZ8iA+5cSd/XS+zvzYb9Pfv7ovSg7+u5Al/CJCcnS5JcXV2LOJLCMXfuXNWvX1916tQp6lAKzZ49exQQEKCpU6dqyJAhGj16tDZu3FjUYRWK6tWr69dff9Wff/4pSTp58qSOHDly2z+yZnP+/HklJiaqbt261jZHR0fVrFlTR44cKcLI7o3k5GRZLJYScdUoMzNTM2bMUJcuXUxx+yOQk5K2r5fY35sN+3v298VZSd/XcwW+BDEMQwsWLFD16tVVuXLlog7nru3YsUMnTpzQlClTijqUQnX+/Hn98MMP6tSpk7p166Zjx45p3rx5cnR0VKtWrYo6vLvy1FNPKTk5Wa+88ors7OyUmZmpPn36qEWLFkUdWqFKTEyUJLm7u9u0u7u7Kz4+vggiunfS0tK0cOFChYSEmH6HLt28ImZvb68OHToUdSjAHSlp+3qJ/b0Zsb9nf1+clfR9PQV8CfLFF1/ojz/+0DvvvFPUody1+Ph4zZ8/X2PHjlWpUqWKOpxClZmZqcDAQPXr10+S5O/vr1OnTikqKsr0O/SdO3dq27Ztevnll1WpUiWdPHlS8+fPtw5uU9JYLBabz4ZhFFEk90ZGRoamT58uwzA0ZMiQog7nrsXGxmrdunWKiIjI9t0BZlGS9vUS+3uzYn/P/r64ehD29RTwJcSXX36pvXv3auLEiXrooYeKOpy7Fhsbq8uXL+vNN9+0tmVmZurw4cNav369Fi5caLpRTrN4enrKz8/Pps3Pz0+7du0qoogKz7///W899dRTCgkJkSRVrlxZFy5c0IoVK0rUDt3Dw0OSrCPvZklKSsp2lt6sMjIyNG3aNF24cEHjx483/dl4STp8+LCSkpJsnkfNzMzUV199pXXr1unTTz8twuiA2ytp+3qJ/b1Zsb9nf19cPQj7egp4kzMMQ19++aV2796tt99+u8QMjlK7dm198MEHNm0zZ85UxYoV9dRTT5l2Zy5J1apVsz4zluXPP/9U+fLliyiiwnP9+vVs342dnV2JO1Pt7e0tDw8PRUdHy9/fX9LNHeChQ4fUv3//Io7u7mXtzOPi4jRhwgSVLVu2qEMqFC1btlTt2rVt2iZNmqSWLVuqTZs2RRQVcHsldV8vsb83K/b37O+LqwdhX08Bb3JffPGFtm/frtGjR8vZ2dn6rI6Li4upb0VzdnbO9mxf6dKlVbZsWdM/89epUyeNGzdO3333nZo3b65jx45p06ZNev7554s6tLvWsGFDfffdd/Ly8pKfn59OnjypNWvWmPIPZmpqquLi4qyfz58/r5MnT8rV1VVeXl7q2LGjli9frgoVKsjHx0fLly9X6dKlTfH8X165eXp6aurUqTpx4oTeeOMNZWZmWv+uuLq6ysGheO82bve93Xpw4uDgIA8PjxL5Pm2UHCV1Xy+xvzcr9vfs74vSg76vtxgl7VTZA6Z37945tg8bNqxE3cIkSW+//baqVKmi8PDwog7lru3du1cLFy5UXFycvL291alTJz3xxBNFHdZdS0lJUWRkpHbv3q3Lly+rXLlyCgkJUc+ePYv1jiAnBw8e1MSJE7O1t2rVSsOHD5dhGPr222+1ceNGXbt2TVWrVtWzzz5rigPOvHLr1auXRowYkeN8EyZMUK1ate51eHfldt/brYYPH66OHTuqU6dO9yM84I48SPt6if29GbC/Z39flB70fT0FPAAAAAAAJmDeB4sAAAAAAHiAUMADAAAAAGACFPAAAAAAAJgABTwAAAAAACZAAQ8AAAAAgAlQwAMAAAAAYAIU8AAAAAAAmAAFPAAAAAAAJkABDwAAAACACVDAAwAAAABgAhTwAAAAAACYAAU8AAAAAAAm8P8AQLfQ0YvDHqQAAAAASUVORK5CYII=","text/plain":["<Figure size 1200x500 with 2 Axes>"]},"metadata":{},"output_type":"display_data"}],"source":["plot_history(history)"]},{"cell_type":"markdown","metadata":{"id":"WEjXm8RpBqST"},"source":["#### 3.4 Long Short Term-Memory (LSTM)"]},{"cell_type":"code","execution_count":10,"metadata":{"executionInfo":{"elapsed":45,"status":"ok","timestamp":1679696224713,"user":{"displayName":"Rafaela Queiroz F. Cordeiro","userId":"18015024781067059704"},"user_tz":420},"id":"9kjJmOTEpSkO"},"outputs":[{"name":"stderr","output_type":"stream","text":["2023-03-28 23:57:31.392326: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations:  AVX2 FMA\n","To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags.\n"]}],"source":["# In case if I want to rerun the model, clear the session to prevent overfitting\n","from keras.backend import clear_session\n","clear_session()"]},{"cell_type":"code","execution_count":16,"metadata":{"executionInfo":{"elapsed":2208,"status":"ok","timestamp":1679696226887,"user":{"displayName":"Rafaela Queiroz F. Cordeiro","userId":"18015024781067059704"},"user_tz":420},"id":"l61l2okngoBQ"},"outputs":[],"source":["# Hyperparameters of the model\n","vocab_size = 66660 # It should be based in the size of your vocabulary (our vocabulary has the size of ?)\n","embedding_dim = 200\n","max_length = 80 # It is based on the length of the sentences (watch out to avoid overfitting and underfitting)\n","padding_type='post'\n","trunc_type='post'\n","num_classes=3\n","\n","# Tokenize the texts\n","tokenizer = Tokenizer(num_words=vocab_size)\n","tokenizer.fit_on_texts(X_train)\n","word_index = tokenizer.word_index\n","\n","# Convert the text data to sequences\n","X_train_seq = tokenizer.texts_to_sequences(X_train)\n","X_test_seq = tokenizer.texts_to_sequences(X_test)\n","\n","# Pad the sequences\n","X_train_padded = pad_sequences(X_train_seq, maxlen=max_length, padding='post')\n","X_test_padded = pad_sequences(X_test_seq, maxlen=max_length, padding='post')"]},{"cell_type":"code","execution_count":67,"metadata":{"colab":{"background_save":true,"base_uri":"https://localhost:8080/"},"id":"iFR7Mgxin2Wg"},"outputs":[{"name":"stdout","output_type":"stream","text":["Epoch 1/5\n","3511/3511 [==============================] - 1129s 320ms/step - loss: 0.6993 - accuracy: 0.7257\n","Epoch 2/5\n","3511/3511 [==============================] - 850s 242ms/step - loss: 0.5210 - accuracy: 0.8079\n","Epoch 3/5\n","3511/3511 [==============================] - 763s 217ms/step - loss: 0.4439 - accuracy: 0.8342\n","Epoch 4/5\n","3511/3511 [==============================] - 801s 228ms/step - loss: 0.3749 - accuracy: 0.8565\n","Epoch 5/5\n","3511/3511 [==============================] - 1306s 372ms/step - loss: 0.3129 - accuracy: 0.8817\n"]},{"data":{"text/plain":["<bound method Model.summary of <keras.engine.sequential.Sequential object at 0x7fa7f8be4b20>>"]},"execution_count":67,"metadata":{},"output_type":"execute_result"}],"source":["# Define the model\n","model = Sequential()\n","model.add(Embedding(input_dim=vocab_size, output_dim=embedding_dim, input_length=max_length))\n","model.add(LSTM(units=64, dropout=0.2, recurrent_dropout=0.2))\n","model.add(Dense(units=num_classes, activation='softmax'))\n","\n","# Compile the model\n","model.compile(loss='categorical_crossentropy', optimizer='adam', metrics=['accuracy'])\n","\n","# Convert integer labels to one-hot vectors\n","y_train_onehot = to_categorical(y_train)\n","y_train_onehot = y_train_onehot.astype('float32')\n","y_test_onehot = to_categorical(y_test)\n","\n","# # Convert integer labels to one-hot vectors\n","# X_train = X_train_padded.astype('float32')\n","# X_test = X_test_padded.astype('float32')\n","# y_train_onehot = to_categorical(y_train)\n","# y_train_onehot = y_train_onehot.astype('float32')\n","# y_test_onehot = to_categorical(y_test)\n","\n","# Train the model\n","model.fit(X_train_padded, y_train_onehot, batch_size=32, epochs=5, verbose=1)\n","model.summary"]},{"cell_type":"code","execution_count":68,"metadata":{"id":"0ukdY6aFfZgl"},"outputs":[{"name":"stdout","output_type":"stream","text":["1681/1681 [==============================] - 24s 14ms/step\n","The accuracy of the model prediction on the test set is 0.6591526243350817\n"]}],"source":["# Predict the model on the test set\n","prediction = model.predict(X_test_padded)\n","\n","# Get labels based on maximum probability\n","predicted_rating = []\n","for rating in prediction:\n","    predicted_rating.append(np.argmax(rating))\n","\n","print(\"The accuracy of the model prediction on the test set is\", accuracy_score(y_test, predicted_rating))"]},{"cell_type":"markdown","metadata":{"id":"ACb7kWUmwR6Y"},"source":["Using *LSTM* to test reviews from the testing set"]},{"cell_type":"code","execution_count":74,"metadata":{},"outputs":[{"name":"stdout","output_type":"stream","text":["1/1 [==============================] - 0s 34ms/step\n","I did not like this pill\n","Predicted sentiment:  Positive\n","I like this medication a lot\n","Predicted sentiment:  Positive\n","I am feelking sad and tired since taking this pill\n","Predicted sentiment:  Positive\n"]}],"source":["reviews_to_test = [ \"I did not like this pill\",\n","                   \"I like this medication a lot\",\n","                   \"I am feelking sad and tired since taking this pill\"]\n","\n","# define class labels mapping\n","labels_mapping = {0: 'Negative', 1: 'Neutral', 2: 'Positive'}\n","\n","# convert to a sequence\n","sequences = tokenizer.texts_to_sequences(reviews_to_test)\n","\n","# pad the sequence\n","padded = pad_sequences(sequences, padding='post', maxlen=max_length)\n","\n","# Get labels based on probability 1 if p>= 0.5 else 0\n","prediction = model.predict(padded)\n","\n","# map predicted labels to sentiment classes\n","predicted_sentiments = []\n","for p in prediction:\n","    label = int(np.argmax(p))\n","    predicted_sentiments.append(labels_mapping[label])\n","\n","# print predicted sentiment for each review\n","for i in range(len(reviews_to_test)):\n","    print(reviews_to_test[i])\n","    print(\"Predicted sentiment: \", predicted_sentiments[i])"]},{"cell_type":"code","execution_count":64,"metadata":{"id":"Wg67fjUWijA5"},"outputs":[{"name":"stdout","output_type":"stream","text":["1/1 [==============================] - 0s 49ms/step\n","[[0.43407378 0.1146035  0.45132276]]\n"]}],"source":["new_review_1 = \"I did not like this pill!\"\n","new_review_tokens = tokenizer.texts_to_sequences([new_review_1])\n","new_review_1_pad = pad_sequences(new_review_tokens, maxlen=max_length, padding='post')\n","new_review_1_padded = pad_sequences(new_review_tokens, maxlen=max_length, padding='post', truncating='post')\n","new_review_1_padded = new_review_1_padded.astype('float32')\n","predicted_sentiment_prob_1 = model.predict(new_review_1_padded)\n","print(predicted_sentiment_prob_1)"]},{"cell_type":"code","execution_count":63,"metadata":{},"outputs":[{"name":"stdout","output_type":"stream","text":["1/1 [==============================] - 0s 35ms/step\n","[[0.43407378 0.1146035  0.45132276]]\n"]}],"source":["new_review_2 = \"I did like this pill!\"\n","new_review_tokens = tokenizer.texts_to_sequences([new_review_2])\n","new_review_2_pad = pad_sequences(new_review_tokens, maxlen=max_length, padding='post')\n","new_review_2_padded = pad_sequences(new_review_tokens, maxlen=max_length, padding='post', truncating='post')\n","new_review_2_padded = new_review_2_padded.astype('float32')\n","predicted_sentiment_prob_2 = model.predict(new_review_2_padded)\n","print(predicted_sentiment_prob_2)"]},{"cell_type":"code","execution_count":62,"metadata":{},"outputs":[{"name":"stdout","output_type":"stream","text":["1/1 [==============================] - 0s 74ms/step\n","[[0.43407378 0.1146035  0.45132276]]\n"]}],"source":["new_review_3 = \"This pill is horrible!\"\n","new_review_tokens = tokenizer.texts_to_sequences([new_review_3])\n","new_review_3_pad = pad_sequences(new_review_tokens, maxlen=max_length, padding='post')\n","new_review_3_padded = pad_sequences(new_review_tokens, maxlen=max_length, padding='post', truncating='post')\n","new_review_3_padded = new_review_1_padded.astype('float32')\n","predicted_sentiment_prob_3 = model.predict(new_review_3_padded)\n","print(predicted_sentiment_prob_3)"]},{"cell_type":"code","execution_count":57,"metadata":{"id":"Nz8iEJZv02nj"},"outputs":[{"name":"stdout","output_type":"stream","text":["1/1 [==============================] - 0s 48ms/step\n","[[0.43407378 0.1146035  0.45132276]]\n","The sentiment of the text is positive\n"]}],"source":["example_1 = [\"I&#039;ve had the copper coil for about 3 months now. I was really excited at the thought of not taking hormones. I&#039;m good with pain however I nearly fainted with insertion, couldn&#039;t belive how painful it was; the doctor did say it is very painful for some. Well 3 months in, my periods last 11 days and I&#039;m in pain for about 15 days with random twangs especially in the left side and I&#039;m considering whether I want to put up with the intense pain and heavy periods. I&#039;d recommend this 100% to somebody who doesn&#039;t already have heavy painful periods but right now it just isn&#039;t for me\"]\n","\n","example_1_review_tokens = tokenizer.texts_to_sequences(example_1)\n","review_1_padded = pad_sequences(example_1_review_tokens, maxlen=max_length, padding='post')\n","\n","# make prediction\n","prediction = model.predict(review_1_padded)\n","print(prediction)\n","\n","# convert prediction to label\n","predicted_label = np.argmax(prediction, axis=-1)\n","if predicted_label == 0:\n","    print(\"The sentiment of the text is negative\")\n","elif predicted_label == 1:\n","    print(\"The sentiment of the text is neutral\")\n","else:\n","    print(\"The sentiment of the text is positive\")\n"]},{"cell_type":"code","execution_count":65,"metadata":{"id":"6S9V5HJN1V2m"},"outputs":[{"name":"stdout","output_type":"stream","text":["1/1 [==============================] - 0s 36ms/step\n","The sentiment of the text is positive\n"]}],"source":["example_2 = [\"I started off by taking regular Ortho Tri Cyclen, which helped some with my acne, but I also became severely depressed, so I went off it.  After a year and a half of not taking birth control at all, I began taking Ortho Tri Cyclen Lo for acne, and after 4 months my skin is MUCH clearer.  If I do get pimples, they are much smaller and more shallow than before, and go away quicker.  I am pleased!\"]\n","\n","example_2_review_tokens = tokenizer.texts_to_sequences(example_2)\n","review_2_padded = pad_sequences(example_2_review_tokens, maxlen=max_length, padding='post')\n","\n","# make prediction\n","prediction = model.predict(review_2_padded)\n","\n","# convert prediction to label\n","predicted_label = np.argmax(prediction, axis=-1)\n","if predicted_label == 0:\n","    print(\"The sentiment of the text is negative\")\n","elif predicted_label == 1:\n","    print(\"The sentiment of the text is neutral\")\n","else:\n","    print(\"The sentiment of the text is positive\")"]},{"cell_type":"code","execution_count":null,"metadata":{"id":"4iF1Iz0V1tgj"},"outputs":[],"source":["example_3 = [\"BC from below. Rapid weight gain, swelling and bad cycle. Don&#039;t recommend\"]\n","\n","example_3_review_tokens = tokenizer.texts_to_sequences(example_3)\n","review_3_padded= pad_sequences(example_3_review_tokens, maxlen=max_length, padding='post')\n","\n","# make prediction\n","prediction = model.predict(review_3_padded)\n","\n","# convert prediction to label\n","predicted_label = np.argmax(prediction, axis=-1)\n","if predicted_label == 0:\n","    print(\"The sentiment of the text is negative\")\n","elif predicted_label == 1:\n","    print(\"The sentiment of the text is neutral\")\n","else:\n","    print(\"The sentiment of the text is positive\")"]},{"cell_type":"code","execution_count":null,"metadata":{"id":"X9IZ3zuW1-i_"},"outputs":[],"source":["example_4 = [\"I love this birth control, because I hate having my period. I have been on this pill for three years already and they are the best.\"]\n","\n","example_4_review_tokens = tokenizer.texts_to_sequences(example_4)\n","review_4_padded = pad_sequences(example_4_review_tokens, maxlen=max_length, padding='post')\n","\n","# make prediction\n","prediction = model.predict(review_4_padded)\n","\n","# convert prediction to label\n","predicted_label = np.argmax(prediction, axis=-1)\n","if predicted_label == 0:\n","    print(\"The sentiment of the text is negative\")\n","elif predicted_label == 1:\n","    print(\"The sentiment of the text is neutral\")\n","else:\n","    print(\"The sentiment of the text is positive\")"]},{"cell_type":"code","execution_count":null,"metadata":{"id":"J05jEukz2Vdl"},"outputs":[],"source":["example_5 = [\"I absolutely love this product and recommend to everyone. I know everyone&#039;s body is different, so it is not for everyone, but it is not the medicines fault. I have NO negative symptoms since I started this. I used to have heavy periods, terrible cramps and headaches, and my periods were super long. I now have super consistent 3-4 LIGHT periods, no cramps or headaches. I have been on this pill for over a year now and have no desire to switch\"]\n","\n","example_5_review_tokens = tokenizer.texts_to_sequences(example_5)\n","review_5_padded = pad_sequences(example_5_review_tokens, maxlen=max_length, padding='post')\n","\n","# make prediction\n","prediction = model.predict(review_5_padded)\n","\n","# convert prediction to label\n","predicted_label = np.argmax(prediction, axis=-1)\n","if predicted_label == 0:\n","    print(\"The sentiment of the text is negative\")\n","elif predicted_label == 1:\n","    print(\"The sentiment of the text is neutral\")\n","else:\n","    print(\"The sentiment of the text is positive\")"]},{"cell_type":"code","execution_count":null,"metadata":{"id":"WOIVCRqL7FHC"},"outputs":[],"source":["example_6 = [\"I have been on this birth control for one cycle. After reading some of the reviews on this type and similar birth controls I was a bit apprehensive to start. Im giving this birth control a 9 out of 10 as I have not been on it long enough for a 10. So far I love this birth control! My side effects have been so minimal its like Im not even on birth control! I have experienced mild headaches here and there and some nausea but other than that ive been feeling great! I got my period on cue on the third day of the inactive pills and I had no idea it was coming because I had zero pms! My period was very light and I barely had any cramping! I had unprotected sex the first month and obviously didn&#039;t get pregnant so I&#039;m very pleased! Highly recommend\"]\n","\n","example_6_review_tokens = tokenizer.texts_to_sequences(example_6)\n","review_6_padded = pad_sequences(example_6_review_tokens, maxlen=max_length, padding='post')\n","\n","# make prediction\n","prediction = model.predict(review_6_padded)\n","\n","# convert prediction to label\n","predicted_label = np.argmax(prediction, axis=-1)\n","if predicted_label == 0:\n","    print(\"The sentiment of the text is negative\")\n","elif predicted_label == 1:\n","    print(\"The sentiment of the text is neutral\")\n","else:\n","    print(\"The sentiment of the text is positive\")"]},{"cell_type":"code","execution_count":null,"metadata":{"id":"57AmGcra7FFK"},"outputs":[],"source":["example_7 = [\"I started on the oral contraceptive pill but got horrible nausea and vomiting, so this was my next option. NuvaRing has been fantastic, it has a fraction of the hormones and I get no nausea. Insertion and removing is really not a hassle, and you cannot feel it at all. For me, it beats the pill and vomiting, along with all the other symptoms. I haven&#039;t had any noticeable mood issues or migraines. I would recommend it to anyone!\"]\n","\n","example_7_review_tokens = tokenizer.texts_to_sequences(example_7)\n","review_7_padded = pad_sequences(example_7_review_tokens, maxlen=max_length, padding='post')\n","\n","# make prediction\n","prediction = model.predict(review_7_padded)\n","\n","# convert prediction to label\n","predicted_label = np.argmax(prediction, axis=-1)\n","if predicted_label == 0:\n","    print(\"The sentiment of the text is negative\")\n","elif predicted_label == 1:\n","    print(\"The sentiment of the text is neutral\")\n","else:\n","    print(\"The sentiment of the text is positive\")"]},{"cell_type":"code","execution_count":null,"metadata":{"id":"Scq9Y1gj7E_S"},"outputs":[],"source":["example_8 = [\"I have been on the shot for 1 year. I have not had any trouble with acne or weight gain. My sex drive is just as high as usual. I have been depressed but depression has been a life long battle for me. The only real con I can think of is that for the first 3-5 days after I get the shot I am a crazy person. The mood swings are 100 times worse then PMS after those first few days. The rest of my three months are normal. Just hoping for the best with my bones.\"]\n","\n","example_8_review_tokens = tokenizer.texts_to_sequences(example_8)\n","review_8_padded = pad_sequences(example_8_review_tokens, maxlen=max_length, padding='post')\n","\n","# make prediction\n","prediction = model.predict(review_8_padded)\n","\n","# convert prediction to label\n","predicted_label = np.argmax(prediction, axis=-1)\n","if predicted_label == 0:\n","    print(\"The sentiment of the text is negative\")\n","elif predicted_label == 1:\n","    print(\"The sentiment of the text is neutral\")\n","else:\n","    print(\"The sentiment of the text is positive\")"]},{"cell_type":"code","execution_count":null,"metadata":{"id":"WvWDV-AF7E8V"},"outputs":[],"source":["example_9 = [\"Switched from my old BC to yaz and immediately felt a difference the first few days I was SO  emotional. VERY significantly so. By the end of the first week I was full on depressed and it continued on to the second week. Everything triggered me. :(  but I like the sticker on the top with the days.\"]\n","\n","example_9_review_tokens = tokenizer.texts_to_sequences(example_9)\n","review_9_padded = pad_sequences(example_9_review_tokens, maxlen=max_length, padding='post')\n","\n","# make prediction\n","prediction = model.predict(review_9_padded)\n","\n","# convert prediction to label\n","predicted_label = np.argmax(prediction, axis=-1)\n","if predicted_label == 0:\n","    print(\"The sentiment of the text is negative\")\n","elif predicted_label == 1:\n","    print(\"The sentiment of the text is neutral\")\n","else:\n","    print(\"The sentiment of the text is positive\")"]},{"cell_type":"code","execution_count":null,"metadata":{"id":"-YWBrtHC7E5y"},"outputs":[],"source":["example_10 = [\"I&#039;ve been on this birth control for a year and I have to say that it&#039;s very practical for my body. I used to get brutal cramps and long periods. No matter what type of birth control I had debilitating cramps. Lolo has caused my cramps to stop and my period is consistent and lasts 2 days. Only downside is I did get some acne, but the acne may be just due to something else since I&#039;ve dealt with breakouts prior to trying lolo. Also my boobs grew a lot which I&#039;m happy about. This medication is not for everyone, if you experience adverse effects stop taking it immediately as it does not agree with every woman.\"]\n","\n","example_10_review_tokens = tokenizer.texts_to_sequences(example_10)\n","review_10_padded = pad_sequences(example_10_review_tokens, maxlen=max_length, padding='post')\n","\n","# make prediction\n","prediction = model.predict(review_10_padded)\n","\n","# convert prediction to label\n","predicted_label = np.argmax(prediction, axis=-1)\n","if predicted_label == 0:\n","    print(\"The sentiment of the text is negative\")\n","elif predicted_label == 1:\n","    print(\"The sentiment of the text is neutral\")\n","else:\n","    print(\"The sentiment of the text is positive\")"]},{"cell_type":"code","execution_count":null,"metadata":{"id":"_PTz8ftNDRz3"},"outputs":[],"source":["# In case if I want to rerun the model, clear the session to prevent overfitting\n","from keras.backend import clear_session\n","clear_session()"]},{"cell_type":"markdown","metadata":{"id":"X0A12oFRCQ65"},"source":["3.5 LSTM and Word2Vec"]},{"cell_type":"code","execution_count":null,"metadata":{"id":"Vpxmr1XVFtKE"},"outputs":[],"source":["# Load the dataframe\n","train_birth_control_df = pd.read_csv('/content/drive/Othercomputers/My MacBook Pro/Sentiment-Analysis-of-Medication-Reviews-Project/birth_control_sample_dataset_processed.csv', sep=',')\n","train_birth_control_df.head()"]},{"cell_type":"code","execution_count":null,"metadata":{"id":"d_4hpHBBCP-v"},"outputs":[],"source":["from keras.preprocessing.text import Tokenizer # This code was adapted from this site: https://realpython.com/python-keras-text-classification/\n","\n","X_train_review_words = train_birth_control_df.review_words\n","X_test_review_words = test_birth_control_df['review']\n","\n","tokenizer = Tokenizer(num_words=5000)\n","tokenizer.fit_on_texts(X_train_review_words)\n","\n","X_train = tokenizer.texts_to_sequences(X_train_review_words)\n","X_test = tokenizer.texts_to_sequences(X_test_review_words)\n","\n","vocab_size = len(tokenizer.word_index) + 1  # Adding 1 because of reserved 0 index\n","\n","print(X_train_review_words[2])\n","print(X_train[2])"]},{"cell_type":"code","execution_count":null,"metadata":{"id":"a2qOqPL4H-KB"},"outputs":[],"source":["for word in ['the', 'all', 'happy', 'sad']:\n","  print('{}: {}'.format(word, tokenizer.word_index[word])) # The number corresponds to the word values from the dictionary tokenizer.word_index."]},{"cell_type":"code","execution_count":null,"metadata":{"id":"8wo4hSEfIQVv"},"outputs":[],"source":["# Padding the sequences of the corpus\n","maxlen = 100\n","\n","X_train = pad_sequences(X_train, padding='post', maxlen=maxlen)\n","X_test = pad_sequences(X_test, padding='post', maxlen=maxlen)"]},{"cell_type":"code","execution_count":null,"metadata":{"id":"i2I70yEOI7gY"},"outputs":[],"source":["X_train[0, :]"]},{"cell_type":"code","execution_count":null,"metadata":{"id":"NgnHSnH-I1ro"},"outputs":[],"source":["X_train[1, :]"]},{"cell_type":"code","execution_count":null,"metadata":{"id":"Ktm3Jp0-JTaq"},"outputs":[],"source":["from keras.models import Sequential\n","from keras import layers\n","\n","vocab_size = 16000 # It should be based in the size of your vocabulary (our vocabulary has the size of 15906)\n","embedding_dim = 50 # embedding_dim = 200 (before)\n","max_length = 100 # It is based on the length of the sentences (watch out to avoid overfitting and underfitting)\n","padding_type='post'\n","trunc_type='post'\n","num_classes=3\n","\n","\n","model = Sequential()\n","model.add(layers.Embedding(input_dim=vocab_size, \n","                           output_dim=embedding_dim, \n","                           input_length=maxlen))\n","model.add(layers.Flatten())\n","model.add(layers.Dense(10, activation='relu'))\n","model.add(layers.Dense(1, activation='softmax'))\n","model.compile(optimizer='adam',\n","              loss='categorical_crossentropy',\n","              metrics=['accuracy'])\n","model.summary()"]},{"cell_type":"code","execution_count":null,"metadata":{"id":"zoIrH9AFK5l-"},"outputs":[],"source":["history = model.fit(X_train, y_train,\n","                    epochs=20,\n","                    verbose=False,\n","                    validation_data=(X_test, y_test),\n","                    batch_size=10)\n","loss, accuracy = model.evaluate(X_train, y_train, verbose=False)\n","print(\"Training Accuracy: {:.4f}\".format(accuracy))\n","loss, accuracy = model.evaluate(X_test, y_test, verbose=False)\n","print(\"Testing Accuracy:  {:.4f}\".format(accuracy))\n","plot_history(history)"]}],"metadata":{"colab":{"name":"","version":""},"kernelspec":{"display_name":"base","language":"python","name":"python3"},"language_info":{"codemirror_mode":{"name":"ipython","version":3},"file_extension":".py","mimetype":"text/x-python","name":"python","nbconvert_exporter":"python","pygments_lexer":"ipython3","version":"3.9.13"}},"nbformat":4,"nbformat_minor":0}