FOTR_SentimentAnalysis.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[nltk_data] Downloading collection 'popular'\n",
      "[nltk_data]    | \n",
      "[nltk_data]    | Downloading package cmudict to\n",
      "[nltk_data]    |     C:\\Users\\Nick\\AppData\\Roaming\\nltk_data...\n",
      "[nltk_data]    |   Package cmudict is already up-to-date!\n",
      "[nltk_data]    | Downloading package gazetteers to\n",
      "[nltk_data]    |     C:\\Users\\Nick\\AppData\\Roaming\\nltk_data...\n",
      "[nltk_data]    |   Package gazetteers is already up-to-date!\n",
      "[nltk_data]    | Downloading package genesis to\n",
      "[nltk_data]    |     C:\\Users\\Nick\\AppData\\Roaming\\nltk_data...\n",
      "[nltk_data]    |   Package genesis is already up-to-date!\n",
      "[nltk_data]    | Downloading package gutenberg to\n",
      "[nltk_data]    |     C:\\Users\\Nick\\AppData\\Roaming\\nltk_data...\n",
      "[nltk_data]    |   Package gutenberg is already up-to-date!\n",
      "[nltk_data]    | Downloading package inaugural to\n",
      "[nltk_data]    |     C:\\Users\\Nick\\AppData\\Roaming\\nltk_data...\n",
      "[nltk_data]    |   Package inaugural is already up-to-date!\n",
      "[nltk_data]    | Downloading package movie_reviews to\n",
      "[nltk_data]    |     C:\\Users\\Nick\\AppData\\Roaming\\nltk_data...\n",
      "[nltk_data]    |   Package movie_reviews is already up-to-date!\n",
      "[nltk_data]    | Downloading package names to\n",
      "[nltk_data]    |     C:\\Users\\Nick\\AppData\\Roaming\\nltk_data...\n",
      "[nltk_data]    |   Package names is already up-to-date!\n",
      "[nltk_data]    | Downloading package shakespeare to\n",
      "[nltk_data]    |     C:\\Users\\Nick\\AppData\\Roaming\\nltk_data...\n",
      "[nltk_data]    |   Package shakespeare is already up-to-date!\n",
      "[nltk_data]    | Downloading package stopwords to\n",
      "[nltk_data]    |     C:\\Users\\Nick\\AppData\\Roaming\\nltk_data...\n",
      "[nltk_data]    |   Package stopwords is already up-to-date!\n",
      "[nltk_data]    | Downloading package treebank to\n",
      "[nltk_data]    |     C:\\Users\\Nick\\AppData\\Roaming\\nltk_data...\n",
      "[nltk_data]    |   Package treebank is already up-to-date!\n",
      "[nltk_data]    | Downloading package twitter_samples to\n",
      "[nltk_data]    |     C:\\Users\\Nick\\AppData\\Roaming\\nltk_data...\n",
      "[nltk_data]    |   Package twitter_samples is already up-to-date!\n",
      "[nltk_data]    | Downloading package omw to\n",
      "[nltk_data]    |     C:\\Users\\Nick\\AppData\\Roaming\\nltk_data...\n",
      "[nltk_data]    |   Package omw is already up-to-date!\n",
      "[nltk_data]    | Downloading package wordnet to\n",
      "[nltk_data]    |     C:\\Users\\Nick\\AppData\\Roaming\\nltk_data...\n",
      "[nltk_data]    |   Package wordnet is already up-to-date!\n",
      "[nltk_data]    | Downloading package wordnet_ic to\n",
      "[nltk_data]    |     C:\\Users\\Nick\\AppData\\Roaming\\nltk_data...\n",
      "[nltk_data]    |   Package wordnet_ic is already up-to-date!\n",
      "[nltk_data]    | Downloading package words to\n",
      "[nltk_data]    |     C:\\Users\\Nick\\AppData\\Roaming\\nltk_data...\n",
      "[nltk_data]    |   Package words is already up-to-date!\n",
      "[nltk_data]    | Downloading package maxent_ne_chunker to\n",
      "[nltk_data]    |     C:\\Users\\Nick\\AppData\\Roaming\\nltk_data...\n",
      "[nltk_data]    |   Package maxent_ne_chunker is already up-to-date!\n",
      "[nltk_data]    | Downloading package punkt to\n",
      "[nltk_data]    |     C:\\Users\\Nick\\AppData\\Roaming\\nltk_data...\n",
      "[nltk_data]    |   Package punkt is already up-to-date!\n",
      "[nltk_data]    | Downloading package snowball_data to\n",
      "[nltk_data]    |     C:\\Users\\Nick\\AppData\\Roaming\\nltk_data...\n",
      "[nltk_data]    |   Package snowball_data is already up-to-date!\n",
      "[nltk_data]    | Downloading package averaged_perceptron_tagger to\n",
      "[nltk_data]    |     C:\\Users\\Nick\\AppData\\Roaming\\nltk_data...\n",
      "[nltk_data]    |   Package averaged_perceptron_tagger is already up-\n",
      "[nltk_data]    |       to-date!\n",
      "[nltk_data]    | \n",
      "[nltk_data]  Done downloading collection popular\n"
     ]
    }
   ],
   "source": [
    "import pandas as pd\n",
    "import nltk\n",
    "from nltk import word_tokenize\n",
    "from nltk.tokenize import sent_tokenize, word_tokenize\n",
    "from nltk.text import Text\n",
    "from nltk.corpus import brown\n",
    "import requests\n",
    "from bs4 import BeautifulSoup\n",
    "nltk.download('popular')\n",
    "pd.set_option('display.max_rows', 500)\n",
    "pd.set_option('display.max_columns', 500)\n",
    "pd.set_option('display.width', 1000)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 381,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<!DOCTYPE html>\n",
      "<html lang=\"en\">\n",
      "<!-- __ _ _ _ __| |_ (_)__ _____\n",
      "    / _` | '_/ _| ' \\| |\\ V / -_)\n",
      "    \\__,_|_| \\__|_||_|_| \\_/\\___| -->\n",
      "  <head>\n",
      "    <title>Full text of &quot;The Lord Of The Rings series&quot;</title>\n",
      "\n",
      "          <meta name=\"viewport\" content=\"width=device-width, initial-scale=1.0\"/>\n",
      "    \n",
      "        <meta name=\"google-site-verification\" content=\"Q2YSouphkkgHkFNP7FgAkc4TmBs1Gmag3uGNndb53B8\" />\n",
      "    <meta name=\"google-site-verification\" content=\"bpjKvUvsX0lxfmjg19TLblckWkDpnptZEYsBntApxUk\" />\n",
      "\n",
      "    <script>\n",
      "/* @licstart  The following is the entire license notice for the\n",
      " * JavaScript code in this page.\n",
      " *\n",
      " * This program is free software: you can redistribute it and/or modify\n",
      " * it under the terms of the GNU Affero General Public License as published by\n",
      " * the Free Software Foundation, either version 3 of the License, or\n",
      " * (at your option) any later version.\n",
      " *\n",
      " * This program is distributed in the hope that it will be useful,\n",
      " * but WITHOUT ANY WARRANTY; without even the implied warranty of\n",
      " * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the\n",
      " * GNU Affero General Public License for more details.\n",
      " *\n",
      " * You should have received a copy of the GNU Affero General Public License\n",
      " * along with this program.  If not, see <http://www.gnu.org/licenses/>.\n",
      " *\n",
      " * @licend  The above is the entire license notice\n",
      " * for the JavaScript code in this page.\n",
      " */\n",
      "</script>\n",
      "        <script>window.archive_setup=[]</script>\n",
      "    <meta charset=\"UTF-8\">\n",
      "    <script src=\"//archive.org/includes/jquery-1.10.2.min.js?v1.10.2\" type=\"text/javascript\"></script>\n",
      "            <script src=\"//archive.org/includes/analytics.js?v=748273f8\" type=\"text/javascript\"></script>\n",
      "    <script>\n",
      "      'use strict';\n",
      "      if ('archive_analytics' in window) {\n",
      "                  archive_analytics.service = \"ao_2\";\n",
      "        \n",
      "        archive_analytics.send_pageview_on_load({\"mediaType\":\"texts\"});\n",
      "\n",
      "        archive_analytics.process_url_events(window.location);\n",
      "      }\n",
      "    </script>\n",
      "        <script s\n"
     ]
    }
   ],
   "source": [
    "r = requests.get('https://archive.org/stream/TheLordOfTheRing1TheFellowshipOfTheRing/The+Lord+Of+The+Ring+1-The+Fellowship+Of+The+Ring_djvu.txt')\n",
    "\n",
    "# Setting the correct text encoding of the HTML page\n",
    "r.encoding = 'utf-8'\n",
    "\n",
    "# Extracting the HTML from the request object\n",
    "html = r.text\n",
    "\n",
    "# Printing the first 2000 characters in html\n",
    "print(html[:2000])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 382,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Creating a BeautifulSoup object from the HTML\n",
    "soup = BeautifulSoup(html, 'html.parser')\n",
    "\n",
    "# Getting the text out of the soup\n",
    "text = soup.get_text()\n",
    "\n",
    "# Fixing mistakes in text\n",
    "text = text.replace('N®menor', 'Numenor')\n",
    "text = text.replace('H®rin', 'Hurin')\n",
    "text = text.replace('T®rin', 'Turin')\n",
    "text = text.replace('D®nedain', 'Dunedain')\n",
    "text = text.replace('ores', 'orcs')\n",
    "text = text.replace('Khazad-dym', 'Khazad-dum')\n",
    "text = text.replace('And®ril', 'Anduril') \n",
    "text = text.replace('Lothlurien', 'Lothlorien')\n",
    "text = text.replace('Lurien', 'Lorien')\n",
    "text = text.replace('Forcst', 'Forest')\n",
    "\n",
    "\n",
    "# Entire Book\n",
    "FOTR = text[16363:1021945]\n",
    "\n",
    "  ## PROLONGUE ##\n",
    "\n",
    "# Concerning Hobbits\n",
    "p1 = FOTR[0:19430] \n",
    "\n",
    "# Concerning Pipe-weed\n",
    "p2 = FOTR[19436:22791]\n",
    "\n",
    "# Of the Ordering of the Shire\n",
    "p3 = FOTR[22797:27493]\n",
    "\n",
    "# Of the Finding of the Ring\n",
    "p4 = FOTR[27499:36075]\n",
    "\n",
    "# NOTE ON THE SHIRE RECORDS\n",
    "p5 = FOTR[36081:41360]\n",
    "\n",
    "  ## BOOK 1 ##\n",
    "\n",
    "# A Long Expected Party\n",
    "b1_1 = FOTR[41381:96384]\n",
    "\n",
    "# The Shadow of the Past\n",
    "b1_2 = FOTR[96390:156092]\n",
    "\n",
    "# Three is Company\n",
    "b1_3 = FOTR[156098:207998]\n",
    "\n",
    "# A Short Cut to Mushrooms\n",
    "b1_4 = FOTR[208004:239136]\n",
    "\n",
    "# A Conspiracy Unmasked\n",
    "b1_5 = FOTR[239142:266920]\n",
    "\n",
    "# The Old Forest\n",
    "b1_6 = FOTR[266925:302280]\n",
    "\n",
    "# In the House of Tom Bombadil\n",
    "b1_7 = FOTR[302286:332328]\n",
    "\n",
    "# Fog on the Barrow-Downs \n",
    "b1_8 = FOTR[332334:368502]\n",
    "\n",
    "# At the Sign of The Prancing Pony \n",
    "b1_9 = FOTR[368508:402448]\n",
    "\n",
    "# Strider\n",
    "b1_10 = FOTR[402453:433350]\n",
    "\n",
    "# A Knife in the Dark\n",
    "b1_11 = FOTR[433356:484433]\n",
    "\n",
    "# Flight to the Ford\n",
    "b1_12 = FOTR[484438:531603]\n",
    "\n",
    "  ## BOOK 2 ##\n",
    "    \n",
    "# Many Meetings\n",
    "b2_1 = FOTR[531624:580340]\n",
    "\n",
    "# The Council of Elrond\n",
    "b2_2 = FOTR[580347:666210]\n",
    "\n",
    "# The Ring Goes South\n",
    "b2_3 = FOTR[666215:723395]\n",
    "\n",
    "# A Journey in the Dark\n",
    "b2_4 = FOTR[723400:785328]\n",
    "\n",
    "# The Bridge of Khazad-dum\n",
    "b2_5 = FOTR[785334:814630]\n",
    "\n",
    "# Lothlorien\n",
    "b2_6 = FOTR[814636:865064]\n",
    "\n",
    "# The Mirror of Galadriel\n",
    "b2_7 = FOTR[865069:901276]\n",
    "\n",
    "# Farewell to Lorien\n",
    "b2_8 = FOTR[901282:934245]\n",
    "\n",
    "# The Great River\n",
    "b2_9 = FOTR[934251:972952]\n",
    "\n",
    "#The Breaking of the Fellowship \n",
    "b2_10 = FOTR[972958:]\n",
    "\n",
    "\n",
    "#print(b1_6)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 337,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "972970"
      ]
     },
     "execution_count": 337,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Find specific spots in the text\n",
    "FOTR.find(\"The Breaking of the Fellowship\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 383,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Text</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1. Concerning Hobbits \\n\\n\\n\\nThis book is lar...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>2. Concerning Pipe-weed \\n\\n\\n\\nThere is anoth...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>3. Of the Ordering of the Shire \\n\\n\\n\\nThe Sh...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>4. Of the Finding of the Ring \\n\\n\\n\\nAs is to...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>NOTE ON THE SHIRE RECORDS \\n\\n\\n\\nAt the end o...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>Chapter 1 . A Long-expected Party \\n\\nWhen Mr....</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>Chapter 2 . The Shadow of the Past \\n\\n\\n\\nThe...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>Chapter 3 . Three is Company \\n\\n\\n\\n'You ough...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>Chapter 4 . A Short Cut to Mushrooms \\n\\n\\n\\nI...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>Chapter 5 . A Conspiracy Unmasked \\n\\n\\n\\n'Now...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>Chapter 6. The Old Forest \\n\\n\\n\\nFrodo woke s...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11</th>\n",
       "      <td>Chapter 7 . In the House of Tom Bombadil \\n\\n\\...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12</th>\n",
       "      <td>Chapter 8 . Fog on the Barrow-Downs \\n\\n\\n\\nTh...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13</th>\n",
       "      <td>Chapter 9 . At the Sign of \\n\\n\\n\\nThe Prancin...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14</th>\n",
       "      <td>Chapter 10. Strider \\n\\n\\n\\nFrodo, Pippin, and...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15</th>\n",
       "      <td>Chapter 11. A Knife in the Dark \\n\\n\\n\\nAs the...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16</th>\n",
       "      <td>Chapter 12 . Flight to the Ford \\n\\n\\n\\nWhen F...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>17</th>\n",
       "      <td>Chapter 1 . Many Meetings \\n\\nFrodo woke and f...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>18</th>\n",
       "      <td>Chapter 2 . The Council of Elrond \\n\\n\\n\\nNext...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>19</th>\n",
       "      <td>Chapter 3 . The Ring Goes South \\n\\n\\n\\nLater ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>20</th>\n",
       "      <td>Chapter 4 . A Journey in the Dark \\n\\n\\n\\nIt w...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>21</th>\n",
       "      <td>Chapter 5 . The Bridge of Khazad-dum \\n\\n\\n\\nT...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>22</th>\n",
       "      <td>Chapter 6 . Lothlorien \\n\\n\\n\\n'Alas! I Fear w...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>23</th>\n",
       "      <td>Chapter 7 . The Mirror of Galadriel \\n\\n\\n\\nTh...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>24</th>\n",
       "      <td>Chapter 8 . Farewell to Lorien \\n\\n\\n\\nThat ni...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25</th>\n",
       "      <td>Chapter 9 . The Great River \\n\\n\\n\\nFrodo was ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>26</th>\n",
       "      <td>Chapter 10. The Breaking of the Fellowship \\n\\...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                 Text\n",
       "0   1. Concerning Hobbits \\n\\n\\n\\nThis book is lar...\n",
       "1   2. Concerning Pipe-weed \\n\\n\\n\\nThere is anoth...\n",
       "2   3. Of the Ordering of the Shire \\n\\n\\n\\nThe Sh...\n",
       "3   4. Of the Finding of the Ring \\n\\n\\n\\nAs is to...\n",
       "4   NOTE ON THE SHIRE RECORDS \\n\\n\\n\\nAt the end o...\n",
       "5   Chapter 1 . A Long-expected Party \\n\\nWhen Mr....\n",
       "6   Chapter 2 . The Shadow of the Past \\n\\n\\n\\nThe...\n",
       "7   Chapter 3 . Three is Company \\n\\n\\n\\n'You ough...\n",
       "8   Chapter 4 . A Short Cut to Mushrooms \\n\\n\\n\\nI...\n",
       "9   Chapter 5 . A Conspiracy Unmasked \\n\\n\\n\\n'Now...\n",
       "10  Chapter 6. The Old Forest \\n\\n\\n\\nFrodo woke s...\n",
       "11  Chapter 7 . In the House of Tom Bombadil \\n\\n\\...\n",
       "12  Chapter 8 . Fog on the Barrow-Downs \\n\\n\\n\\nTh...\n",
       "13  Chapter 9 . At the Sign of \\n\\n\\n\\nThe Prancin...\n",
       "14  Chapter 10. Strider \\n\\n\\n\\nFrodo, Pippin, and...\n",
       "15  Chapter 11. A Knife in the Dark \\n\\n\\n\\nAs the...\n",
       "16  Chapter 12 . Flight to the Ford \\n\\n\\n\\nWhen F...\n",
       "17  Chapter 1 . Many Meetings \\n\\nFrodo woke and f...\n",
       "18  Chapter 2 . The Council of Elrond \\n\\n\\n\\nNext...\n",
       "19  Chapter 3 . The Ring Goes South \\n\\n\\n\\nLater ...\n",
       "20  Chapter 4 . A Journey in the Dark \\n\\n\\n\\nIt w...\n",
       "21  Chapter 5 . The Bridge of Khazad-dum \\n\\n\\n\\nT...\n",
       "22  Chapter 6 . Lothlorien \\n\\n\\n\\n'Alas! I Fear w...\n",
       "23  Chapter 7 . The Mirror of Galadriel \\n\\n\\n\\nTh...\n",
       "24  Chapter 8 . Farewell to Lorien \\n\\n\\n\\nThat ni...\n",
       "25  Chapter 9 . The Great River \\n\\n\\n\\nFrodo was ...\n",
       "26  Chapter 10. The Breaking of the Fellowship \\n\\..."
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "texts = [p1,p2,p3,p4,p5,b1_1,b1_2,b1_3,b1_4,b1_5,b1_6,b1_7,b1_8,b1_9,b1_10,b1_11,b1_12,b2_1,b2_2,b2_3,\n",
    "         b2_4,b2_5,b2_6,b2_7,b2_8,b2_9,b2_10]\n",
    "\n",
    "data = pd.DataFrame(data=[text for text in texts], columns=['Text'])\n",
    "\n",
    "display(data.head(27))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 384,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Text</th>\n",
       "      <th>Chapters</th>\n",
       "      <th>Book</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1. Concerning Hobbits \\n\\n\\n\\nThis book is lar...</td>\n",
       "      <td>Concerning Hobbits</td>\n",
       "      <td>Prologue</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>2. Concerning Pipe-weed \\n\\n\\n\\nThere is anoth...</td>\n",
       "      <td>Concerning Pipe-weed</td>\n",
       "      <td>Prologue</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>3. Of the Ordering of the Shire \\n\\n\\n\\nThe Sh...</td>\n",
       "      <td>Of the Ordering of the Shire</td>\n",
       "      <td>Prologue</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>4. Of the Finding of the Ring \\n\\n\\n\\nAs is to...</td>\n",
       "      <td>Of the Finding of the Ring</td>\n",
       "      <td>Prologue</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>NOTE ON THE SHIRE RECORDS \\n\\n\\n\\nAt the end o...</td>\n",
       "      <td>NOTE ON THE SHIRE RECORDS</td>\n",
       "      <td>Prologue</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>Chapter 1 . A Long-expected Party \\n\\nWhen Mr....</td>\n",
       "      <td>A Long-expected Party</td>\n",
       "      <td>Book I</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>Chapter 2 . The Shadow of the Past \\n\\n\\n\\nThe...</td>\n",
       "      <td>The Shadow of the Past</td>\n",
       "      <td>Book I</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>Chapter 3 . Three is Company \\n\\n\\n\\n'You ough...</td>\n",
       "      <td>Three is Company</td>\n",
       "      <td>Book I</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>Chapter 4 . A Short Cut to Mushrooms \\n\\n\\n\\nI...</td>\n",
       "      <td>A Short Cut to Mushrooms</td>\n",
       "      <td>Book I</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>Chapter 5 . A Conspiracy Unmasked \\n\\n\\n\\n'Now...</td>\n",
       "      <td>A Conspiracy Unmasked</td>\n",
       "      <td>Book I</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>Chapter 6. The Old Forest \\n\\n\\n\\nFrodo woke s...</td>\n",
       "      <td>The Old Forest</td>\n",
       "      <td>Book I</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11</th>\n",
       "      <td>Chapter 7 . In the House of Tom Bombadil \\n\\n\\...</td>\n",
       "      <td>In the House of Tom Bombadil</td>\n",
       "      <td>Book I</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12</th>\n",
       "      <td>Chapter 8 . Fog on the Barrow-Downs \\n\\n\\n\\nTh...</td>\n",
       "      <td>Fog on the Barrow-Downs</td>\n",
       "      <td>Book I</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13</th>\n",
       "      <td>Chapter 9 . At the Sign of \\n\\n\\n\\nThe Prancin...</td>\n",
       "      <td>At the Sign of The Prancing Pony</td>\n",
       "      <td>Book I</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14</th>\n",
       "      <td>Chapter 10. Strider \\n\\n\\n\\nFrodo, Pippin, and...</td>\n",
       "      <td>Strider</td>\n",
       "      <td>Book I</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15</th>\n",
       "      <td>Chapter 11. A Knife in the Dark \\n\\n\\n\\nAs the...</td>\n",
       "      <td>A Knife in the Dark</td>\n",
       "      <td>Book I</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16</th>\n",
       "      <td>Chapter 12 . Flight to the Ford \\n\\n\\n\\nWhen F...</td>\n",
       "      <td>Flight to the Ford</td>\n",
       "      <td>Book I</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>17</th>\n",
       "      <td>Chapter 1 . Many Meetings \\n\\nFrodo woke and f...</td>\n",
       "      <td>Many Meetings</td>\n",
       "      <td>Book II</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>18</th>\n",
       "      <td>Chapter 2 . The Council of Elrond \\n\\n\\n\\nNext...</td>\n",
       "      <td>The Council of Elrond</td>\n",
       "      <td>Book II</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>19</th>\n",
       "      <td>Chapter 3 . The Ring Goes South \\n\\n\\n\\nLater ...</td>\n",
       "      <td>The Ring Goes South</td>\n",
       "      <td>Book II</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>20</th>\n",
       "      <td>Chapter 4 . A Journey in the Dark \\n\\n\\n\\nIt w...</td>\n",
       "      <td>A Journey in the Dark</td>\n",
       "      <td>Book II</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>21</th>\n",
       "      <td>Chapter 5 . The Bridge of Khazad-dum \\n\\n\\n\\nT...</td>\n",
       "      <td>The Bridge of Khazad-dum</td>\n",
       "      <td>Book II</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>22</th>\n",
       "      <td>Chapter 6 . Lothlorien \\n\\n\\n\\n'Alas! I Fear w...</td>\n",
       "      <td>Lothlorien</td>\n",
       "      <td>Book II</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>23</th>\n",
       "      <td>Chapter 7 . The Mirror of Galadriel \\n\\n\\n\\nTh...</td>\n",
       "      <td>The Mirror of Galadriel</td>\n",
       "      <td>Book II</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>24</th>\n",
       "      <td>Chapter 8 . Farewell to Lorien \\n\\n\\n\\nThat ni...</td>\n",
       "      <td>Farewell to Lorien</td>\n",
       "      <td>Book II</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25</th>\n",
       "      <td>Chapter 9 . The Great River \\n\\n\\n\\nFrodo was ...</td>\n",
       "      <td>The Great River</td>\n",
       "      <td>Book II</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>26</th>\n",
       "      <td>Chapter 10. The Breaking of the Fellowship \\n\\...</td>\n",
       "      <td>The Breaking of the Fellowship</td>\n",
       "      <td>Book II</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                 Text                          Chapters      Book\n",
       "0   1. Concerning Hobbits \\n\\n\\n\\nThis book is lar...                Concerning Hobbits  Prologue\n",
       "1   2. Concerning Pipe-weed \\n\\n\\n\\nThere is anoth...              Concerning Pipe-weed  Prologue\n",
       "2   3. Of the Ordering of the Shire \\n\\n\\n\\nThe Sh...      Of the Ordering of the Shire  Prologue\n",
       "3   4. Of the Finding of the Ring \\n\\n\\n\\nAs is to...        Of the Finding of the Ring  Prologue\n",
       "4   NOTE ON THE SHIRE RECORDS \\n\\n\\n\\nAt the end o...         NOTE ON THE SHIRE RECORDS  Prologue\n",
       "5   Chapter 1 . A Long-expected Party \\n\\nWhen Mr....             A Long-expected Party    Book I\n",
       "6   Chapter 2 . The Shadow of the Past \\n\\n\\n\\nThe...            The Shadow of the Past    Book I\n",
       "7   Chapter 3 . Three is Company \\n\\n\\n\\n'You ough...                  Three is Company    Book I\n",
       "8   Chapter 4 . A Short Cut to Mushrooms \\n\\n\\n\\nI...          A Short Cut to Mushrooms    Book I\n",
       "9   Chapter 5 . A Conspiracy Unmasked \\n\\n\\n\\n'Now...             A Conspiracy Unmasked    Book I\n",
       "10  Chapter 6. The Old Forest \\n\\n\\n\\nFrodo woke s...                    The Old Forest    Book I\n",
       "11  Chapter 7 . In the House of Tom Bombadil \\n\\n\\...      In the House of Tom Bombadil    Book I\n",
       "12  Chapter 8 . Fog on the Barrow-Downs \\n\\n\\n\\nTh...           Fog on the Barrow-Downs    Book I\n",
       "13  Chapter 9 . At the Sign of \\n\\n\\n\\nThe Prancin...  At the Sign of The Prancing Pony    Book I\n",
       "14  Chapter 10. Strider \\n\\n\\n\\nFrodo, Pippin, and...                           Strider    Book I\n",
       "15  Chapter 11. A Knife in the Dark \\n\\n\\n\\nAs the...               A Knife in the Dark    Book I\n",
       "16  Chapter 12 . Flight to the Ford \\n\\n\\n\\nWhen F...                Flight to the Ford    Book I\n",
       "17  Chapter 1 . Many Meetings \\n\\nFrodo woke and f...                     Many Meetings   Book II\n",
       "18  Chapter 2 . The Council of Elrond \\n\\n\\n\\nNext...             The Council of Elrond   Book II\n",
       "19  Chapter 3 . The Ring Goes South \\n\\n\\n\\nLater ...               The Ring Goes South   Book II\n",
       "20  Chapter 4 . A Journey in the Dark \\n\\n\\n\\nIt w...             A Journey in the Dark   Book II\n",
       "21  Chapter 5 . The Bridge of Khazad-dum \\n\\n\\n\\nT...          The Bridge of Khazad-dum   Book II\n",
       "22  Chapter 6 . Lothlorien \\n\\n\\n\\n'Alas! I Fear w...                        Lothlorien   Book II\n",
       "23  Chapter 7 . The Mirror of Galadriel \\n\\n\\n\\nTh...           The Mirror of Galadriel   Book II\n",
       "24  Chapter 8 . Farewell to Lorien \\n\\n\\n\\nThat ni...                Farewell to Lorien   Book II\n",
       "25  Chapter 9 . The Great River \\n\\n\\n\\nFrodo was ...                   The Great River   Book II\n",
       "26  Chapter 10. The Breaking of the Fellowship \\n\\...    The Breaking of the Fellowship   Book II"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "chapters = ['Concerning Hobbits', 'Concerning Pipe-weed', 'Of the Ordering of the Shire', 'Of the Finding of the Ring',\n",
    "           'NOTE ON THE SHIRE RECORDS', 'A Long-expected Party', 'The Shadow of the Past', 'Three is Company',\n",
    "           'A Short Cut to Mushrooms', 'A Conspiracy Unmasked', 'The Old Forest', 'In the House of Tom Bombadil',\n",
    "           'Fog on the Barrow-Downs', 'At the Sign of The Prancing Pony', 'Strider', 'A Knife in the Dark', \n",
    "           'Flight to the Ford', 'Many Meetings', 'The Council of Elrond', 'The Ring Goes South', 'A Journey in the Dark',\n",
    "           'The Bridge of Khazad-dum', 'Lothlorien', 'The Mirror of Galadriel', 'Farewell to Lorien', 'The Great River',\n",
    "           'The Breaking of the Fellowship']\n",
    "\n",
    "book = ['Prologue', 'Prologue', 'Prologue', 'Prologue','Prologue', 'Book I', 'Book I', 'Book I','Book I', \n",
    "        'Book I', 'Book I', 'Book I','Book I', 'Book I', 'Book I', 'Book I', 'Book I', 'Book II', 'Book II', \n",
    "        'Book II', 'Book II','Book II', 'Book II', 'Book II', 'Book II', 'Book II','Book II']\n",
    "\n",
    "data1 = pd.DataFrame(chapters, columns=['Chapters'])\n",
    "\n",
    "data1_1 = pd.merge(data, data1, left_index=True, right_index=True)\n",
    "\n",
    "data1_2 = pd.DataFrame(book, columns=['Book'])\n",
    "\n",
    "data2 = pd.merge(data1_1, data1_2, left_index=True, right_index=True)\n",
    "\n",
    "\n",
    "display(data2.head(27))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 385,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Volume</th>\n",
       "      <th>Book</th>\n",
       "      <th>Chapters</th>\n",
       "      <th>Text</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>FOTR</td>\n",
       "      <td>Prologue</td>\n",
       "      <td>Concerning Hobbits</td>\n",
       "      <td>1. Concerning Hobbits \\n\\n\\n\\nThis book is lar...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>FOTR</td>\n",
       "      <td>Prologue</td>\n",
       "      <td>Concerning Pipe-weed</td>\n",
       "      <td>2. Concerning Pipe-weed \\n\\n\\n\\nThere is anoth...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>FOTR</td>\n",
       "      <td>Prologue</td>\n",
       "      <td>Of the Ordering of the Shire</td>\n",
       "      <td>3. Of the Ordering of the Shire \\n\\n\\n\\nThe Sh...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>FOTR</td>\n",
       "      <td>Prologue</td>\n",
       "      <td>Of the Finding of the Ring</td>\n",
       "      <td>4. Of the Finding of the Ring \\n\\n\\n\\nAs is to...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>FOTR</td>\n",
       "      <td>Prologue</td>\n",
       "      <td>NOTE ON THE SHIRE RECORDS</td>\n",
       "      <td>NOTE ON THE SHIRE RECORDS \\n\\n\\n\\nAt the end o...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>FOTR</td>\n",
       "      <td>Book I</td>\n",
       "      <td>A Long-expected Party</td>\n",
       "      <td>Chapter 1 . A Long-expected Party \\n\\nWhen Mr....</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>FOTR</td>\n",
       "      <td>Book I</td>\n",
       "      <td>The Shadow of the Past</td>\n",
       "      <td>Chapter 2 . The Shadow of the Past \\n\\n\\n\\nThe...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>FOTR</td>\n",
       "      <td>Book I</td>\n",
       "      <td>Three is Company</td>\n",
       "      <td>Chapter 3 . Three is Company \\n\\n\\n\\n'You ough...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>FOTR</td>\n",
       "      <td>Book I</td>\n",
       "      <td>A Short Cut to Mushrooms</td>\n",
       "      <td>Chapter 4 . A Short Cut to Mushrooms \\n\\n\\n\\nI...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>FOTR</td>\n",
       "      <td>Book I</td>\n",
       "      <td>A Conspiracy Unmasked</td>\n",
       "      <td>Chapter 5 . A Conspiracy Unmasked \\n\\n\\n\\n'Now...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>FOTR</td>\n",
       "      <td>Book I</td>\n",
       "      <td>The Old Forest</td>\n",
       "      <td>Chapter 6. The Old Forest \\n\\n\\n\\nFrodo woke s...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11</th>\n",
       "      <td>FOTR</td>\n",
       "      <td>Book I</td>\n",
       "      <td>In the House of Tom Bombadil</td>\n",
       "      <td>Chapter 7 . In the House of Tom Bombadil \\n\\n\\...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12</th>\n",
       "      <td>FOTR</td>\n",
       "      <td>Book I</td>\n",
       "      <td>Fog on the Barrow-Downs</td>\n",
       "      <td>Chapter 8 . Fog on the Barrow-Downs \\n\\n\\n\\nTh...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13</th>\n",
       "      <td>FOTR</td>\n",
       "      <td>Book I</td>\n",
       "      <td>At the Sign of The Prancing Pony</td>\n",
       "      <td>Chapter 9 . At the Sign of \\n\\n\\n\\nThe Prancin...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14</th>\n",
       "      <td>FOTR</td>\n",
       "      <td>Book I</td>\n",
       "      <td>Strider</td>\n",
       "      <td>Chapter 10. Strider \\n\\n\\n\\nFrodo, Pippin, and...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15</th>\n",
       "      <td>FOTR</td>\n",
       "      <td>Book I</td>\n",
       "      <td>A Knife in the Dark</td>\n",
       "      <td>Chapter 11. A Knife in the Dark \\n\\n\\n\\nAs the...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16</th>\n",
       "      <td>FOTR</td>\n",
       "      <td>Book I</td>\n",
       "      <td>Flight to the Ford</td>\n",
       "      <td>Chapter 12 . Flight to the Ford \\n\\n\\n\\nWhen F...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>17</th>\n",
       "      <td>FOTR</td>\n",
       "      <td>Book II</td>\n",
       "      <td>Many Meetings</td>\n",
       "      <td>Chapter 1 . Many Meetings \\n\\nFrodo woke and f...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>18</th>\n",
       "      <td>FOTR</td>\n",
       "      <td>Book II</td>\n",
       "      <td>The Council of Elrond</td>\n",
       "      <td>Chapter 2 . The Council of Elrond \\n\\n\\n\\nNext...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>19</th>\n",
       "      <td>FOTR</td>\n",
       "      <td>Book II</td>\n",
       "      <td>The Ring Goes South</td>\n",
       "      <td>Chapter 3 . The Ring Goes South \\n\\n\\n\\nLater ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>20</th>\n",
       "      <td>FOTR</td>\n",
       "      <td>Book II</td>\n",
       "      <td>A Journey in the Dark</td>\n",
       "      <td>Chapter 4 . A Journey in the Dark \\n\\n\\n\\nIt w...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>21</th>\n",
       "      <td>FOTR</td>\n",
       "      <td>Book II</td>\n",
       "      <td>The Bridge of Khazad-dum</td>\n",
       "      <td>Chapter 5 . The Bridge of Khazad-dum \\n\\n\\n\\nT...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>22</th>\n",
       "      <td>FOTR</td>\n",
       "      <td>Book II</td>\n",
       "      <td>Lothlorien</td>\n",
       "      <td>Chapter 6 . Lothlorien \\n\\n\\n\\n'Alas! I Fear w...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>23</th>\n",
       "      <td>FOTR</td>\n",
       "      <td>Book II</td>\n",
       "      <td>The Mirror of Galadriel</td>\n",
       "      <td>Chapter 7 . The Mirror of Galadriel \\n\\n\\n\\nTh...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>24</th>\n",
       "      <td>FOTR</td>\n",
       "      <td>Book II</td>\n",
       "      <td>Farewell to Lorien</td>\n",
       "      <td>Chapter 8 . Farewell to Lorien \\n\\n\\n\\nThat ni...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25</th>\n",
       "      <td>FOTR</td>\n",
       "      <td>Book II</td>\n",
       "      <td>The Great River</td>\n",
       "      <td>Chapter 9 . The Great River \\n\\n\\n\\nFrodo was ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>26</th>\n",
       "      <td>FOTR</td>\n",
       "      <td>Book II</td>\n",
       "      <td>The Breaking of the Fellowship</td>\n",
       "      <td>Chapter 10. The Breaking of the Fellowship \\n\\...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   Volume      Book                          Chapters                                               Text\n",
       "0    FOTR  Prologue                Concerning Hobbits  1. Concerning Hobbits \\n\\n\\n\\nThis book is lar...\n",
       "1    FOTR  Prologue              Concerning Pipe-weed  2. Concerning Pipe-weed \\n\\n\\n\\nThere is anoth...\n",
       "2    FOTR  Prologue      Of the Ordering of the Shire  3. Of the Ordering of the Shire \\n\\n\\n\\nThe Sh...\n",
       "3    FOTR  Prologue        Of the Finding of the Ring  4. Of the Finding of the Ring \\n\\n\\n\\nAs is to...\n",
       "4    FOTR  Prologue         NOTE ON THE SHIRE RECORDS  NOTE ON THE SHIRE RECORDS \\n\\n\\n\\nAt the end o...\n",
       "5    FOTR    Book I             A Long-expected Party  Chapter 1 . A Long-expected Party \\n\\nWhen Mr....\n",
       "6    FOTR    Book I            The Shadow of the Past  Chapter 2 . The Shadow of the Past \\n\\n\\n\\nThe...\n",
       "7    FOTR    Book I                  Three is Company  Chapter 3 . Three is Company \\n\\n\\n\\n'You ough...\n",
       "8    FOTR    Book I          A Short Cut to Mushrooms  Chapter 4 . A Short Cut to Mushrooms \\n\\n\\n\\nI...\n",
       "9    FOTR    Book I             A Conspiracy Unmasked  Chapter 5 . A Conspiracy Unmasked \\n\\n\\n\\n'Now...\n",
       "10   FOTR    Book I                    The Old Forest  Chapter 6. The Old Forest \\n\\n\\n\\nFrodo woke s...\n",
       "11   FOTR    Book I      In the House of Tom Bombadil  Chapter 7 . In the House of Tom Bombadil \\n\\n\\...\n",
       "12   FOTR    Book I           Fog on the Barrow-Downs  Chapter 8 . Fog on the Barrow-Downs \\n\\n\\n\\nTh...\n",
       "13   FOTR    Book I  At the Sign of The Prancing Pony  Chapter 9 . At the Sign of \\n\\n\\n\\nThe Prancin...\n",
       "14   FOTR    Book I                           Strider  Chapter 10. Strider \\n\\n\\n\\nFrodo, Pippin, and...\n",
       "15   FOTR    Book I               A Knife in the Dark  Chapter 11. A Knife in the Dark \\n\\n\\n\\nAs the...\n",
       "16   FOTR    Book I                Flight to the Ford  Chapter 12 . Flight to the Ford \\n\\n\\n\\nWhen F...\n",
       "17   FOTR   Book II                     Many Meetings  Chapter 1 . Many Meetings \\n\\nFrodo woke and f...\n",
       "18   FOTR   Book II             The Council of Elrond  Chapter 2 . The Council of Elrond \\n\\n\\n\\nNext...\n",
       "19   FOTR   Book II               The Ring Goes South  Chapter 3 . The Ring Goes South \\n\\n\\n\\nLater ...\n",
       "20   FOTR   Book II             A Journey in the Dark  Chapter 4 . A Journey in the Dark \\n\\n\\n\\nIt w...\n",
       "21   FOTR   Book II          The Bridge of Khazad-dum  Chapter 5 . The Bridge of Khazad-dum \\n\\n\\n\\nT...\n",
       "22   FOTR   Book II                        Lothlorien  Chapter 6 . Lothlorien \\n\\n\\n\\n'Alas! I Fear w...\n",
       "23   FOTR   Book II           The Mirror of Galadriel  Chapter 7 . The Mirror of Galadriel \\n\\n\\n\\nTh...\n",
       "24   FOTR   Book II                Farewell to Lorien  Chapter 8 . Farewell to Lorien \\n\\n\\n\\nThat ni...\n",
       "25   FOTR   Book II                   The Great River  Chapter 9 . The Great River \\n\\n\\n\\nFrodo was ...\n",
       "26   FOTR   Book II    The Breaking of the Fellowship  Chapter 10. The Breaking of the Fellowship \\n\\..."
      ]
     },
     "execution_count": 385,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data2['Volume']='FOTR'\n",
    "data2 = data2[['Volume', 'Book', 'Chapters', 'Text']]\n",
    "data2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 386,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>compound</th>\n",
       "      <th>neg</th>\n",
       "      <th>neu</th>\n",
       "      <th>pos</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0.133970</td>\n",
       "      <td>0.043674</td>\n",
       "      <td>0.880891</td>\n",
       "      <td>0.075450</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0.239865</td>\n",
       "      <td>0.014870</td>\n",
       "      <td>0.903565</td>\n",
       "      <td>0.081609</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0.042571</td>\n",
       "      <td>0.041147</td>\n",
       "      <td>0.903382</td>\n",
       "      <td>0.055471</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>0.075828</td>\n",
       "      <td>0.075471</td>\n",
       "      <td>0.841000</td>\n",
       "      <td>0.083485</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0.115635</td>\n",
       "      <td>0.025703</td>\n",
       "      <td>0.914108</td>\n",
       "      <td>0.060162</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>0.092960</td>\n",
       "      <td>0.050563</td>\n",
       "      <td>0.854325</td>\n",
       "      <td>0.095113</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>0.023151</td>\n",
       "      <td>0.080364</td>\n",
       "      <td>0.835787</td>\n",
       "      <td>0.083856</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>0.062997</td>\n",
       "      <td>0.039382</td>\n",
       "      <td>0.887917</td>\n",
       "      <td>0.072693</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>0.011641</td>\n",
       "      <td>0.054406</td>\n",
       "      <td>0.882502</td>\n",
       "      <td>0.063086</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>0.083426</td>\n",
       "      <td>0.049687</td>\n",
       "      <td>0.853551</td>\n",
       "      <td>0.096768</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>0.050078</td>\n",
       "      <td>0.059573</td>\n",
       "      <td>0.859174</td>\n",
       "      <td>0.081251</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11</th>\n",
       "      <td>0.109893</td>\n",
       "      <td>0.051551</td>\n",
       "      <td>0.848711</td>\n",
       "      <td>0.099737</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12</th>\n",
       "      <td>0.063105</td>\n",
       "      <td>0.056751</td>\n",
       "      <td>0.864729</td>\n",
       "      <td>0.078517</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13</th>\n",
       "      <td>0.063927</td>\n",
       "      <td>0.044007</td>\n",
       "      <td>0.889855</td>\n",
       "      <td>0.066140</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14</th>\n",
       "      <td>0.031716</td>\n",
       "      <td>0.064495</td>\n",
       "      <td>0.854428</td>\n",
       "      <td>0.081069</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15</th>\n",
       "      <td>-0.017724</td>\n",
       "      <td>0.078997</td>\n",
       "      <td>0.860830</td>\n",
       "      <td>0.060170</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16</th>\n",
       "      <td>-0.039676</td>\n",
       "      <td>0.077316</td>\n",
       "      <td>0.866580</td>\n",
       "      <td>0.056111</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>17</th>\n",
       "      <td>0.092482</td>\n",
       "      <td>0.047418</td>\n",
       "      <td>0.856683</td>\n",
       "      <td>0.095900</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>18</th>\n",
       "      <td>0.007270</td>\n",
       "      <td>0.074671</td>\n",
       "      <td>0.853501</td>\n",
       "      <td>0.071823</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>19</th>\n",
       "      <td>0.000085</td>\n",
       "      <td>0.064984</td>\n",
       "      <td>0.863647</td>\n",
       "      <td>0.068535</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>20</th>\n",
       "      <td>-0.018587</td>\n",
       "      <td>0.068742</td>\n",
       "      <td>0.871763</td>\n",
       "      <td>0.059500</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>21</th>\n",
       "      <td>-0.073328</td>\n",
       "      <td>0.097166</td>\n",
       "      <td>0.859562</td>\n",
       "      <td>0.043271</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>22</th>\n",
       "      <td>0.050481</td>\n",
       "      <td>0.052154</td>\n",
       "      <td>0.882938</td>\n",
       "      <td>0.064901</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>23</th>\n",
       "      <td>0.045263</td>\n",
       "      <td>0.061203</td>\n",
       "      <td>0.860048</td>\n",
       "      <td>0.078742</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>24</th>\n",
       "      <td>0.091091</td>\n",
       "      <td>0.050251</td>\n",
       "      <td>0.866813</td>\n",
       "      <td>0.082930</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25</th>\n",
       "      <td>-0.020113</td>\n",
       "      <td>0.065061</td>\n",
       "      <td>0.890092</td>\n",
       "      <td>0.044852</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>26</th>\n",
       "      <td>-0.025731</td>\n",
       "      <td>0.099378</td>\n",
       "      <td>0.834948</td>\n",
       "      <td>0.065674</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "    compound       neg       neu       pos\n",
       "0   0.133970  0.043674  0.880891  0.075450\n",
       "1   0.239865  0.014870  0.903565  0.081609\n",
       "2   0.042571  0.041147  0.903382  0.055471\n",
       "3   0.075828  0.075471  0.841000  0.083485\n",
       "4   0.115635  0.025703  0.914108  0.060162\n",
       "5   0.092960  0.050563  0.854325  0.095113\n",
       "6   0.023151  0.080364  0.835787  0.083856\n",
       "7   0.062997  0.039382  0.887917  0.072693\n",
       "8   0.011641  0.054406  0.882502  0.063086\n",
       "9   0.083426  0.049687  0.853551  0.096768\n",
       "10  0.050078  0.059573  0.859174  0.081251\n",
       "11  0.109893  0.051551  0.848711  0.099737\n",
       "12  0.063105  0.056751  0.864729  0.078517\n",
       "13  0.063927  0.044007  0.889855  0.066140\n",
       "14  0.031716  0.064495  0.854428  0.081069\n",
       "15 -0.017724  0.078997  0.860830  0.060170\n",
       "16 -0.039676  0.077316  0.866580  0.056111\n",
       "17  0.092482  0.047418  0.856683  0.095900\n",
       "18  0.007270  0.074671  0.853501  0.071823\n",
       "19  0.000085  0.064984  0.863647  0.068535\n",
       "20 -0.018587  0.068742  0.871763  0.059500\n",
       "21 -0.073328  0.097166  0.859562  0.043271\n",
       "22  0.050481  0.052154  0.882938  0.064901\n",
       "23  0.045263  0.061203  0.860048  0.078742\n",
       "24  0.091091  0.050251  0.866813  0.082930\n",
       "25 -0.020113  0.065061  0.890092  0.044852\n",
       "26 -0.025731  0.099378  0.834948  0.065674"
      ]
     },
     "execution_count": 386,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import pandas as pd\n",
    "from nltk import tokenize\n",
    "from vaderSentiment.vaderSentiment import SentimentIntensityAnalyzer \n",
    "\n",
    "chapters = [p1,p2,p3,p4,p5,b1_1,b1_2,b1_3,b1_4,b1_5,b1_6,b1_7,b1_8,b1_9,b1_10,b1_11,b1_12,b2_1,b2_2,b2_3,\n",
    "         b2_4,b2_5,b2_6,b2_7,b2_8,b2_9,b2_10]\n",
    "\n",
    "analyzer = SentimentIntensityAnalyzer()\n",
    "\n",
    "sentiments_list = list()\n",
    "\n",
    "for chapter in chapters:\n",
    "    sentence_list = tokenize.sent_tokenize(chapter)\n",
    "    sentiments = {'compound': 0.0, 'neg': 0.0, 'neu': 0.0, 'pos': 0.0}\n",
    "        \n",
    "    for sentence in sentence_list:\n",
    "        vs = analyzer.polarity_scores(sentence)\n",
    "        sentiments['compound'] += vs['compound']\n",
    "        sentiments['neg'] += vs['neg']\n",
    "        sentiments['neu'] += vs['neu']\n",
    "        sentiments['pos'] += vs['pos']\n",
    "            \n",
    "    sentiments['compound'] = sentiments['compound'] / len(sentence_list)\n",
    "    sentiments['neg'] = sentiments['neg'] / len(sentence_list)\n",
    "    sentiments['neu'] = sentiments['neu'] / len(sentence_list)\n",
    "    sentiments['pos'] = sentiments['pos'] / len(sentence_list)\n",
    "    \n",
    "    sentiments_list.append(sentiments)  # add this line\n",
    "\n",
    "data3 = pd.DataFrame(sentiments_list)  # add this line\n",
    "data3"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 387,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Volume</th>\n",
       "      <th>Book</th>\n",
       "      <th>Chapters</th>\n",
       "      <th>Text</th>\n",
       "      <th>Compound</th>\n",
       "      <th>Negative</th>\n",
       "      <th>Neutral</th>\n",
       "      <th>Positive</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>FOTR</td>\n",
       "      <td>Prologue</td>\n",
       "      <td>Concerning Hobbits</td>\n",
       "      <td>1. Concerning Hobbits \\n\\n\\n\\nThis book is lar...</td>\n",
       "      <td>0.133970</td>\n",
       "      <td>0.043674</td>\n",
       "      <td>0.880891</td>\n",
       "      <td>0.075450</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>FOTR</td>\n",
       "      <td>Prologue</td>\n",
       "      <td>Concerning Pipe-weed</td>\n",
       "      <td>2. Concerning Pipe-weed \\n\\n\\n\\nThere is anoth...</td>\n",
       "      <td>0.239865</td>\n",
       "      <td>0.014870</td>\n",
       "      <td>0.903565</td>\n",
       "      <td>0.081609</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>FOTR</td>\n",
       "      <td>Prologue</td>\n",
       "      <td>Of the Ordering of the Shire</td>\n",
       "      <td>3. Of the Ordering of the Shire \\n\\n\\n\\nThe Sh...</td>\n",
       "      <td>0.042571</td>\n",
       "      <td>0.041147</td>\n",
       "      <td>0.903382</td>\n",
       "      <td>0.055471</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>FOTR</td>\n",
       "      <td>Prologue</td>\n",
       "      <td>Of the Finding of the Ring</td>\n",
       "      <td>4. Of the Finding of the Ring \\n\\n\\n\\nAs is to...</td>\n",
       "      <td>0.075828</td>\n",
       "      <td>0.075471</td>\n",
       "      <td>0.841000</td>\n",
       "      <td>0.083485</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>FOTR</td>\n",
       "      <td>Prologue</td>\n",
       "      <td>NOTE ON THE SHIRE RECORDS</td>\n",
       "      <td>NOTE ON THE SHIRE RECORDS \\n\\n\\n\\nAt the end o...</td>\n",
       "      <td>0.115635</td>\n",
       "      <td>0.025703</td>\n",
       "      <td>0.914108</td>\n",
       "      <td>0.060162</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>FOTR</td>\n",
       "      <td>Book I</td>\n",
       "      <td>A Long-expected Party</td>\n",
       "      <td>Chapter 1 . A Long-expected Party \\n\\nWhen Mr....</td>\n",
       "      <td>0.092960</td>\n",
       "      <td>0.050563</td>\n",
       "      <td>0.854325</td>\n",
       "      <td>0.095113</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>FOTR</td>\n",
       "      <td>Book I</td>\n",
       "      <td>The Shadow of the Past</td>\n",
       "      <td>Chapter 2 . The Shadow of the Past \\n\\n\\n\\nThe...</td>\n",
       "      <td>0.023151</td>\n",
       "      <td>0.080364</td>\n",
       "      <td>0.835787</td>\n",
       "      <td>0.083856</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>FOTR</td>\n",
       "      <td>Book I</td>\n",
       "      <td>Three is Company</td>\n",
       "      <td>Chapter 3 . Three is Company \\n\\n\\n\\n'You ough...</td>\n",
       "      <td>0.062997</td>\n",
       "      <td>0.039382</td>\n",
       "      <td>0.887917</td>\n",
       "      <td>0.072693</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>FOTR</td>\n",
       "      <td>Book I</td>\n",
       "      <td>A Short Cut to Mushrooms</td>\n",
       "      <td>Chapter 4 . A Short Cut to Mushrooms \\n\\n\\n\\nI...</td>\n",
       "      <td>0.011641</td>\n",
       "      <td>0.054406</td>\n",
       "      <td>0.882502</td>\n",
       "      <td>0.063086</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>FOTR</td>\n",
       "      <td>Book I</td>\n",
       "      <td>A Conspiracy Unmasked</td>\n",
       "      <td>Chapter 5 . A Conspiracy Unmasked \\n\\n\\n\\n'Now...</td>\n",
       "      <td>0.083426</td>\n",
       "      <td>0.049687</td>\n",
       "      <td>0.853551</td>\n",
       "      <td>0.096768</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>FOTR</td>\n",
       "      <td>Book I</td>\n",
       "      <td>The Old Forest</td>\n",
       "      <td>Chapter 6. The Old Forest \\n\\n\\n\\nFrodo woke s...</td>\n",
       "      <td>0.050078</td>\n",
       "      <td>0.059573</td>\n",
       "      <td>0.859174</td>\n",
       "      <td>0.081251</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11</th>\n",
       "      <td>FOTR</td>\n",
       "      <td>Book I</td>\n",
       "      <td>In the House of Tom Bombadil</td>\n",
       "      <td>Chapter 7 . In the House of Tom Bombadil \\n\\n\\...</td>\n",
       "      <td>0.109893</td>\n",
       "      <td>0.051551</td>\n",
       "      <td>0.848711</td>\n",
       "      <td>0.099737</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12</th>\n",
       "      <td>FOTR</td>\n",
       "      <td>Book I</td>\n",
       "      <td>Fog on the Barrow-Downs</td>\n",
       "      <td>Chapter 8 . Fog on the Barrow-Downs \\n\\n\\n\\nTh...</td>\n",
       "      <td>0.063105</td>\n",
       "      <td>0.056751</td>\n",
       "      <td>0.864729</td>\n",
       "      <td>0.078517</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13</th>\n",
       "      <td>FOTR</td>\n",
       "      <td>Book I</td>\n",
       "      <td>At the Sign of The Prancing Pony</td>\n",
       "      <td>Chapter 9 . At the Sign of \\n\\n\\n\\nThe Prancin...</td>\n",
       "      <td>0.063927</td>\n",
       "      <td>0.044007</td>\n",
       "      <td>0.889855</td>\n",
       "      <td>0.066140</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14</th>\n",
       "      <td>FOTR</td>\n",
       "      <td>Book I</td>\n",
       "      <td>Strider</td>\n",
       "      <td>Chapter 10. Strider \\n\\n\\n\\nFrodo, Pippin, and...</td>\n",
       "      <td>0.031716</td>\n",
       "      <td>0.064495</td>\n",
       "      <td>0.854428</td>\n",
       "      <td>0.081069</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15</th>\n",
       "      <td>FOTR</td>\n",
       "      <td>Book I</td>\n",
       "      <td>A Knife in the Dark</td>\n",
       "      <td>Chapter 11. A Knife in the Dark \\n\\n\\n\\nAs the...</td>\n",
       "      <td>-0.017724</td>\n",
       "      <td>0.078997</td>\n",
       "      <td>0.860830</td>\n",
       "      <td>0.060170</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16</th>\n",
       "      <td>FOTR</td>\n",
       "      <td>Book I</td>\n",
       "      <td>Flight to the Ford</td>\n",
       "      <td>Chapter 12 . Flight to the Ford \\n\\n\\n\\nWhen F...</td>\n",
       "      <td>-0.039676</td>\n",
       "      <td>0.077316</td>\n",
       "      <td>0.866580</td>\n",
       "      <td>0.056111</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>17</th>\n",
       "      <td>FOTR</td>\n",
       "      <td>Book II</td>\n",
       "      <td>Many Meetings</td>\n",
       "      <td>Chapter 1 . Many Meetings \\n\\nFrodo woke and f...</td>\n",
       "      <td>0.092482</td>\n",
       "      <td>0.047418</td>\n",
       "      <td>0.856683</td>\n",
       "      <td>0.095900</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>18</th>\n",
       "      <td>FOTR</td>\n",
       "      <td>Book II</td>\n",
       "      <td>The Council of Elrond</td>\n",
       "      <td>Chapter 2 . The Council of Elrond \\n\\n\\n\\nNext...</td>\n",
       "      <td>0.007270</td>\n",
       "      <td>0.074671</td>\n",
       "      <td>0.853501</td>\n",
       "      <td>0.071823</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>19</th>\n",
       "      <td>FOTR</td>\n",
       "      <td>Book II</td>\n",
       "      <td>The Ring Goes South</td>\n",
       "      <td>Chapter 3 . The Ring Goes South \\n\\n\\n\\nLater ...</td>\n",
       "      <td>0.000085</td>\n",
       "      <td>0.064984</td>\n",
       "      <td>0.863647</td>\n",
       "      <td>0.068535</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>20</th>\n",
       "      <td>FOTR</td>\n",
       "      <td>Book II</td>\n",
       "      <td>A Journey in the Dark</td>\n",
       "      <td>Chapter 4 . A Journey in the Dark \\n\\n\\n\\nIt w...</td>\n",
       "      <td>-0.018587</td>\n",
       "      <td>0.068742</td>\n",
       "      <td>0.871763</td>\n",
       "      <td>0.059500</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>21</th>\n",
       "      <td>FOTR</td>\n",
       "      <td>Book II</td>\n",
       "      <td>The Bridge of Khazad-dum</td>\n",
       "      <td>Chapter 5 . The Bridge of Khazad-dum \\n\\n\\n\\nT...</td>\n",
       "      <td>-0.073328</td>\n",
       "      <td>0.097166</td>\n",
       "      <td>0.859562</td>\n",
       "      <td>0.043271</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>22</th>\n",
       "      <td>FOTR</td>\n",
       "      <td>Book II</td>\n",
       "      <td>Lothlorien</td>\n",
       "      <td>Chapter 6 . Lothlorien \\n\\n\\n\\n'Alas! I Fear w...</td>\n",
       "      <td>0.050481</td>\n",
       "      <td>0.052154</td>\n",
       "      <td>0.882938</td>\n",
       "      <td>0.064901</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>23</th>\n",
       "      <td>FOTR</td>\n",
       "      <td>Book II</td>\n",
       "      <td>The Mirror of Galadriel</td>\n",
       "      <td>Chapter 7 . The Mirror of Galadriel \\n\\n\\n\\nTh...</td>\n",
       "      <td>0.045263</td>\n",
       "      <td>0.061203</td>\n",
       "      <td>0.860048</td>\n",
       "      <td>0.078742</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>24</th>\n",
       "      <td>FOTR</td>\n",
       "      <td>Book II</td>\n",
       "      <td>Farewell to Lorien</td>\n",
       "      <td>Chapter 8 . Farewell to Lorien \\n\\n\\n\\nThat ni...</td>\n",
       "      <td>0.091091</td>\n",
       "      <td>0.050251</td>\n",
       "      <td>0.866813</td>\n",
       "      <td>0.082930</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25</th>\n",
       "      <td>FOTR</td>\n",
       "      <td>Book II</td>\n",
       "      <td>The Great River</td>\n",
       "      <td>Chapter 9 . The Great River \\n\\n\\n\\nFrodo was ...</td>\n",
       "      <td>-0.020113</td>\n",
       "      <td>0.065061</td>\n",
       "      <td>0.890092</td>\n",
       "      <td>0.044852</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>26</th>\n",
       "      <td>FOTR</td>\n",
       "      <td>Book II</td>\n",
       "      <td>The Breaking of the Fellowship</td>\n",
       "      <td>Chapter 10. The Breaking of the Fellowship \\n\\...</td>\n",
       "      <td>-0.025731</td>\n",
       "      <td>0.099378</td>\n",
       "      <td>0.834948</td>\n",
       "      <td>0.065674</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   Volume      Book                          Chapters                                               Text  Compound  Negative   Neutral  Positive\n",
       "0    FOTR  Prologue                Concerning Hobbits  1. Concerning Hobbits \\n\\n\\n\\nThis book is lar...  0.133970  0.043674  0.880891  0.075450\n",
       "1    FOTR  Prologue              Concerning Pipe-weed  2. Concerning Pipe-weed \\n\\n\\n\\nThere is anoth...  0.239865  0.014870  0.903565  0.081609\n",
       "2    FOTR  Prologue      Of the Ordering of the Shire  3. Of the Ordering of the Shire \\n\\n\\n\\nThe Sh...  0.042571  0.041147  0.903382  0.055471\n",
       "3    FOTR  Prologue        Of the Finding of the Ring  4. Of the Finding of the Ring \\n\\n\\n\\nAs is to...  0.075828  0.075471  0.841000  0.083485\n",
       "4    FOTR  Prologue         NOTE ON THE SHIRE RECORDS  NOTE ON THE SHIRE RECORDS \\n\\n\\n\\nAt the end o...  0.115635  0.025703  0.914108  0.060162\n",
       "5    FOTR    Book I             A Long-expected Party  Chapter 1 . A Long-expected Party \\n\\nWhen Mr....  0.092960  0.050563  0.854325  0.095113\n",
       "6    FOTR    Book I            The Shadow of the Past  Chapter 2 . The Shadow of the Past \\n\\n\\n\\nThe...  0.023151  0.080364  0.835787  0.083856\n",
       "7    FOTR    Book I                  Three is Company  Chapter 3 . Three is Company \\n\\n\\n\\n'You ough...  0.062997  0.039382  0.887917  0.072693\n",
       "8    FOTR    Book I          A Short Cut to Mushrooms  Chapter 4 . A Short Cut to Mushrooms \\n\\n\\n\\nI...  0.011641  0.054406  0.882502  0.063086\n",
       "9    FOTR    Book I             A Conspiracy Unmasked  Chapter 5 . A Conspiracy Unmasked \\n\\n\\n\\n'Now...  0.083426  0.049687  0.853551  0.096768\n",
       "10   FOTR    Book I                    The Old Forest  Chapter 6. The Old Forest \\n\\n\\n\\nFrodo woke s...  0.050078  0.059573  0.859174  0.081251\n",
       "11   FOTR    Book I      In the House of Tom Bombadil  Chapter 7 . In the House of Tom Bombadil \\n\\n\\...  0.109893  0.051551  0.848711  0.099737\n",
       "12   FOTR    Book I           Fog on the Barrow-Downs  Chapter 8 . Fog on the Barrow-Downs \\n\\n\\n\\nTh...  0.063105  0.056751  0.864729  0.078517\n",
       "13   FOTR    Book I  At the Sign of The Prancing Pony  Chapter 9 . At the Sign of \\n\\n\\n\\nThe Prancin...  0.063927  0.044007  0.889855  0.066140\n",
       "14   FOTR    Book I                           Strider  Chapter 10. Strider \\n\\n\\n\\nFrodo, Pippin, and...  0.031716  0.064495  0.854428  0.081069\n",
       "15   FOTR    Book I               A Knife in the Dark  Chapter 11. A Knife in the Dark \\n\\n\\n\\nAs the... -0.017724  0.078997  0.860830  0.060170\n",
       "16   FOTR    Book I                Flight to the Ford  Chapter 12 . Flight to the Ford \\n\\n\\n\\nWhen F... -0.039676  0.077316  0.866580  0.056111\n",
       "17   FOTR   Book II                     Many Meetings  Chapter 1 . Many Meetings \\n\\nFrodo woke and f...  0.092482  0.047418  0.856683  0.095900\n",
       "18   FOTR   Book II             The Council of Elrond  Chapter 2 . The Council of Elrond \\n\\n\\n\\nNext...  0.007270  0.074671  0.853501  0.071823\n",
       "19   FOTR   Book II               The Ring Goes South  Chapter 3 . The Ring Goes South \\n\\n\\n\\nLater ...  0.000085  0.064984  0.863647  0.068535\n",
       "20   FOTR   Book II             A Journey in the Dark  Chapter 4 . A Journey in the Dark \\n\\n\\n\\nIt w... -0.018587  0.068742  0.871763  0.059500\n",
       "21   FOTR   Book II          The Bridge of Khazad-dum  Chapter 5 . The Bridge of Khazad-dum \\n\\n\\n\\nT... -0.073328  0.097166  0.859562  0.043271\n",
       "22   FOTR   Book II                        Lothlorien  Chapter 6 . Lothlorien \\n\\n\\n\\n'Alas! I Fear w...  0.050481  0.052154  0.882938  0.064901\n",
       "23   FOTR   Book II           The Mirror of Galadriel  Chapter 7 . The Mirror of Galadriel \\n\\n\\n\\nTh...  0.045263  0.061203  0.860048  0.078742\n",
       "24   FOTR   Book II                Farewell to Lorien  Chapter 8 . Farewell to Lorien \\n\\n\\n\\nThat ni...  0.091091  0.050251  0.866813  0.082930\n",
       "25   FOTR   Book II                   The Great River  Chapter 9 . The Great River \\n\\n\\n\\nFrodo was ... -0.020113  0.065061  0.890092  0.044852\n",
       "26   FOTR   Book II    The Breaking of the Fellowship  Chapter 10. The Breaking of the Fellowship \\n\\... -0.025731  0.099378  0.834948  0.065674"
      ]
     },
     "execution_count": 387,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "FOTR_sentiments = pd.merge(data2, data3, left_index=True, right_index=True)\n",
    "\n",
    "FOTR_sentiments=FOTR_sentiments.rename(columns={\"compound\": \"Compound\", \"neg\": \"Negative\", \"neu\": \"Neutral\", \"pos\": \"Positive\"})\n",
    "\n",
    "FOTR_sentiments.to_csv(r'C:\\Users\\Nick\\Desktop\\GitProjects\\NLP_projects\\FOTR\\FOTR_sentiments.csv')\n",
    "\n",
    "FOTR_sentiments"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 388,
   "metadata": {},
   "outputs": [],
   "source": [
    "from textblob import TextBlob\n",
    "\n",
    "blob_book = TextBlob(FOTR)\n",
    "blob_p1 = TextBlob(p1)\n",
    "blob_p2 = TextBlob(p2)\n",
    "blob_p3 = TextBlob(p3)\n",
    "blob_p4 = TextBlob(p4)\n",
    "blob_p5 = TextBlob(p5)\n",
    "blob_b1_1 = TextBlob(b1_1)\n",
    "blob_b1_2 = TextBlob(b1_2)\n",
    "blob_b1_3 = TextBlob(b1_3)\n",
    "blob_b1_4 = TextBlob(b1_4)\n",
    "blob_b1_5 = TextBlob(b1_5)\n",
    "blob_b1_6 = TextBlob(b1_6)\n",
    "blob_b1_7 = TextBlob(b1_7)\n",
    "blob_b1_8 = TextBlob(b1_8)\n",
    "blob_b1_9 = TextBlob(b1_9)\n",
    "blob_b1_10 = TextBlob(b1_10)\n",
    "blob_b1_11 = TextBlob(b1_11)\n",
    "blob_b1_12 = TextBlob(b1_12)\n",
    "blob_b2_1 = TextBlob(b2_1)\n",
    "blob_b2_2 = TextBlob(b2_2)\n",
    "blob_b2_3 = TextBlob(b2_3)\n",
    "blob_b2_4 = TextBlob(b2_4)\n",
    "blob_b2_5 = TextBlob(b2_5)\n",
    "blob_b2_6 = TextBlob(b2_6)\n",
    "blob_b2_7 = TextBlob(b2_7)\n",
    "blob_b2_8 = TextBlob(b2_8)\n",
    "blob_b2_9 = TextBlob(b2_9)\n",
    "blob_b2_10 = TextBlob(b2_10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 389,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Polarity</th>\n",
       "      <th>Subjectivity</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0.123098</td>\n",
       "      <td>0.453589</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0.191711</td>\n",
       "      <td>0.472525</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0.097907</td>\n",
       "      <td>0.441715</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>0.041690</td>\n",
       "      <td>0.336673</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0.220372</td>\n",
       "      <td>0.394603</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>0.063388</td>\n",
       "      <td>0.297571</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>0.046675</td>\n",
       "      <td>0.299241</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>0.056428</td>\n",
       "      <td>0.285911</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>0.048184</td>\n",
       "      <td>0.262968</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>0.046885</td>\n",
       "      <td>0.278871</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>0.011858</td>\n",
       "      <td>0.283457</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11</th>\n",
       "      <td>0.078520</td>\n",
       "      <td>0.314967</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12</th>\n",
       "      <td>0.012824</td>\n",
       "      <td>0.315016</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13</th>\n",
       "      <td>0.055907</td>\n",
       "      <td>0.274922</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14</th>\n",
       "      <td>0.018458</td>\n",
       "      <td>0.252212</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15</th>\n",
       "      <td>0.012372</td>\n",
       "      <td>0.298401</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16</th>\n",
       "      <td>0.007771</td>\n",
       "      <td>0.303397</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>17</th>\n",
       "      <td>0.073006</td>\n",
       "      <td>0.287620</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>18</th>\n",
       "      <td>0.040775</td>\n",
       "      <td>0.304679</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>19</th>\n",
       "      <td>0.037318</td>\n",
       "      <td>0.313236</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>20</th>\n",
       "      <td>0.010430</td>\n",
       "      <td>0.304156</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>21</th>\n",
       "      <td>0.014297</td>\n",
       "      <td>0.247021</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>22</th>\n",
       "      <td>0.055373</td>\n",
       "      <td>0.290735</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>23</th>\n",
       "      <td>0.051823</td>\n",
       "      <td>0.331532</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>24</th>\n",
       "      <td>0.062147</td>\n",
       "      <td>0.321413</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25</th>\n",
       "      <td>0.018566</td>\n",
       "      <td>0.306630</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>26</th>\n",
       "      <td>0.003442</td>\n",
       "      <td>0.237485</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "    Polarity  Subjectivity\n",
       "0   0.123098      0.453589\n",
       "1   0.191711      0.472525\n",
       "2   0.097907      0.441715\n",
       "3   0.041690      0.336673\n",
       "4   0.220372      0.394603\n",
       "5   0.063388      0.297571\n",
       "6   0.046675      0.299241\n",
       "7   0.056428      0.285911\n",
       "8   0.048184      0.262968\n",
       "9   0.046885      0.278871\n",
       "10  0.011858      0.283457\n",
       "11  0.078520      0.314967\n",
       "12  0.012824      0.315016\n",
       "13  0.055907      0.274922\n",
       "14  0.018458      0.252212\n",
       "15  0.012372      0.298401\n",
       "16  0.007771      0.303397\n",
       "17  0.073006      0.287620\n",
       "18  0.040775      0.304679\n",
       "19  0.037318      0.313236\n",
       "20  0.010430      0.304156\n",
       "21  0.014297      0.247021\n",
       "22  0.055373      0.290735\n",
       "23  0.051823      0.331532\n",
       "24  0.062147      0.321413\n",
       "25  0.018566      0.306630\n",
       "26  0.003442      0.237485"
      ]
     },
     "execution_count": 389,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "blobs = [blob_p1, blob_p2, blob_p3, blob_p4, blob_p5, blob_b1_1, blob_b1_2, blob_b1_3, blob_b1_4, blob_b1_5,\n",
    "        blob_b1_6, blob_b1_7, blob_b1_8, blob_b1_9, blob_b1_10, blob_b1_11, blob_b1_12, blob_b2_1, blob_b2_2,\n",
    "        blob_b2_3, blob_b2_4, blob_b2_5, blob_b2_6, blob_b2_7, blob_b2_8, blob_b2_9, blob_b2_10]\n",
    "\n",
    "polarity_list = list()\n",
    "subjectivity_list = list()\n",
    "\n",
    "for blob in blobs:\n",
    "    sentence_list1 = blob.sentences\n",
    "    polarity = (0)\n",
    "    subjectivity = (0)\n",
    "        \n",
    "    for sentence in sentence_list1:\n",
    "        pl=sentence.sentiment.polarity\n",
    "        polarity += pl\n",
    "        sb=sentence.sentiment.subjectivity\n",
    "        subjectivity += sb\n",
    "            \n",
    "    polarity = polarity / len(sentence_list1)\n",
    "    subjectivity = subjectivity / len(sentence_list1)\n",
    "\n",
    "    polarity_list.append(polarity)  \n",
    "    subjectivity_list.append(subjectivity)\n",
    "\n",
    "data4 = pd.DataFrame(polarity_list)  \n",
    "data5 = pd.DataFrame(subjectivity_list)  \n",
    "data6 = pd.merge(data4, data5, left_index=True, right_index=True)\n",
    "data6=data6.rename(columns={'0_x': \"Polarity\", '0_y': 'Subjectivity'})\n",
    "\n",
    "data6"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 390,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Volume</th>\n",
       "      <th>Book</th>\n",
       "      <th>Chapters</th>\n",
       "      <th>Text</th>\n",
       "      <th>Compound</th>\n",
       "      <th>Negative</th>\n",
       "      <th>Neutral</th>\n",
       "      <th>Positive</th>\n",
       "      <th>Polarity</th>\n",
       "      <th>Subjectivity</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>FOTR</td>\n",
       "      <td>Prologue</td>\n",
       "      <td>Concerning Hobbits</td>\n",
       "      <td>1. Concerning Hobbits \\n\\n\\n\\nThis book is lar...</td>\n",
       "      <td>0.133970</td>\n",
       "      <td>0.043674</td>\n",
       "      <td>0.880891</td>\n",
       "      <td>0.075450</td>\n",
       "      <td>0.123098</td>\n",
       "      <td>0.453589</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>FOTR</td>\n",
       "      <td>Prologue</td>\n",
       "      <td>Concerning Pipe-weed</td>\n",
       "      <td>2. Concerning Pipe-weed \\n\\n\\n\\nThere is anoth...</td>\n",
       "      <td>0.239865</td>\n",
       "      <td>0.014870</td>\n",
       "      <td>0.903565</td>\n",
       "      <td>0.081609</td>\n",
       "      <td>0.191711</td>\n",
       "      <td>0.472525</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>FOTR</td>\n",
       "      <td>Prologue</td>\n",
       "      <td>Of the Ordering of the Shire</td>\n",
       "      <td>3. Of the Ordering of the Shire \\n\\n\\n\\nThe Sh...</td>\n",
       "      <td>0.042571</td>\n",
       "      <td>0.041147</td>\n",
       "      <td>0.903382</td>\n",
       "      <td>0.055471</td>\n",
       "      <td>0.097907</td>\n",
       "      <td>0.441715</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>FOTR</td>\n",
       "      <td>Prologue</td>\n",
       "      <td>Of the Finding of the Ring</td>\n",
       "      <td>4. Of the Finding of the Ring \\n\\n\\n\\nAs is to...</td>\n",
       "      <td>0.075828</td>\n",
       "      <td>0.075471</td>\n",
       "      <td>0.841000</td>\n",
       "      <td>0.083485</td>\n",
       "      <td>0.041690</td>\n",
       "      <td>0.336673</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>FOTR</td>\n",
       "      <td>Prologue</td>\n",
       "      <td>NOTE ON THE SHIRE RECORDS</td>\n",
       "      <td>NOTE ON THE SHIRE RECORDS \\n\\n\\n\\nAt the end o...</td>\n",
       "      <td>0.115635</td>\n",
       "      <td>0.025703</td>\n",
       "      <td>0.914108</td>\n",
       "      <td>0.060162</td>\n",
       "      <td>0.220372</td>\n",
       "      <td>0.394603</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>FOTR</td>\n",
       "      <td>Book I</td>\n",
       "      <td>A Long-expected Party</td>\n",
       "      <td>Chapter 1 . A Long-expected Party \\n\\nWhen Mr....</td>\n",
       "      <td>0.092960</td>\n",
       "      <td>0.050563</td>\n",
       "      <td>0.854325</td>\n",
       "      <td>0.095113</td>\n",
       "      <td>0.063388</td>\n",
       "      <td>0.297571</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>FOTR</td>\n",
       "      <td>Book I</td>\n",
       "      <td>The Shadow of the Past</td>\n",
       "      <td>Chapter 2 . The Shadow of the Past \\n\\n\\n\\nThe...</td>\n",
       "      <td>0.023151</td>\n",
       "      <td>0.080364</td>\n",
       "      <td>0.835787</td>\n",
       "      <td>0.083856</td>\n",
       "      <td>0.046675</td>\n",
       "      <td>0.299241</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>FOTR</td>\n",
       "      <td>Book I</td>\n",
       "      <td>Three is Company</td>\n",
       "      <td>Chapter 3 . Three is Company \\n\\n\\n\\n'You ough...</td>\n",
       "      <td>0.062997</td>\n",
       "      <td>0.039382</td>\n",
       "      <td>0.887917</td>\n",
       "      <td>0.072693</td>\n",
       "      <td>0.056428</td>\n",
       "      <td>0.285911</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>FOTR</td>\n",
       "      <td>Book I</td>\n",
       "      <td>A Short Cut to Mushrooms</td>\n",
       "      <td>Chapter 4 . A Short Cut to Mushrooms \\n\\n\\n\\nI...</td>\n",
       "      <td>0.011641</td>\n",
       "      <td>0.054406</td>\n",
       "      <td>0.882502</td>\n",
       "      <td>0.063086</td>\n",
       "      <td>0.048184</td>\n",
       "      <td>0.262968</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>FOTR</td>\n",
       "      <td>Book I</td>\n",
       "      <td>A Conspiracy Unmasked</td>\n",
       "      <td>Chapter 5 . A Conspiracy Unmasked \\n\\n\\n\\n'Now...</td>\n",
       "      <td>0.083426</td>\n",
       "      <td>0.049687</td>\n",
       "      <td>0.853551</td>\n",
       "      <td>0.096768</td>\n",
       "      <td>0.046885</td>\n",
       "      <td>0.278871</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>FOTR</td>\n",
       "      <td>Book I</td>\n",
       "      <td>The Old Forest</td>\n",
       "      <td>Chapter 6. The Old Forest \\n\\n\\n\\nFrodo woke s...</td>\n",
       "      <td>0.050078</td>\n",
       "      <td>0.059573</td>\n",
       "      <td>0.859174</td>\n",
       "      <td>0.081251</td>\n",
       "      <td>0.011858</td>\n",
       "      <td>0.283457</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11</th>\n",
       "      <td>FOTR</td>\n",
       "      <td>Book I</td>\n",
       "      <td>In the House of Tom Bombadil</td>\n",
       "      <td>Chapter 7 . In the House of Tom Bombadil \\n\\n\\...</td>\n",
       "      <td>0.109893</td>\n",
       "      <td>0.051551</td>\n",
       "      <td>0.848711</td>\n",
       "      <td>0.099737</td>\n",
       "      <td>0.078520</td>\n",
       "      <td>0.314967</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12</th>\n",
       "      <td>FOTR</td>\n",
       "      <td>Book I</td>\n",
       "      <td>Fog on the Barrow-Downs</td>\n",
       "      <td>Chapter 8 . Fog on the Barrow-Downs \\n\\n\\n\\nTh...</td>\n",
       "      <td>0.063105</td>\n",
       "      <td>0.056751</td>\n",
       "      <td>0.864729</td>\n",
       "      <td>0.078517</td>\n",
       "      <td>0.012824</td>\n",
       "      <td>0.315016</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13</th>\n",
       "      <td>FOTR</td>\n",
       "      <td>Book I</td>\n",
       "      <td>At the Sign of The Prancing Pony</td>\n",
       "      <td>Chapter 9 . At the Sign of \\n\\n\\n\\nThe Prancin...</td>\n",
       "      <td>0.063927</td>\n",
       "      <td>0.044007</td>\n",
       "      <td>0.889855</td>\n",
       "      <td>0.066140</td>\n",
       "      <td>0.055907</td>\n",
       "      <td>0.274922</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14</th>\n",
       "      <td>FOTR</td>\n",
       "      <td>Book I</td>\n",
       "      <td>Strider</td>\n",
       "      <td>Chapter 10. Strider \\n\\n\\n\\nFrodo, Pippin, and...</td>\n",
       "      <td>0.031716</td>\n",
       "      <td>0.064495</td>\n",
       "      <td>0.854428</td>\n",
       "      <td>0.081069</td>\n",
       "      <td>0.018458</td>\n",
       "      <td>0.252212</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15</th>\n",
       "      <td>FOTR</td>\n",
       "      <td>Book I</td>\n",
       "      <td>A Knife in the Dark</td>\n",
       "      <td>Chapter 11. A Knife in the Dark \\n\\n\\n\\nAs the...</td>\n",
       "      <td>-0.017724</td>\n",
       "      <td>0.078997</td>\n",
       "      <td>0.860830</td>\n",
       "      <td>0.060170</td>\n",
       "      <td>0.012372</td>\n",
       "      <td>0.298401</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16</th>\n",
       "      <td>FOTR</td>\n",
       "      <td>Book I</td>\n",
       "      <td>Flight to the Ford</td>\n",
       "      <td>Chapter 12 . Flight to the Ford \\n\\n\\n\\nWhen F...</td>\n",
       "      <td>-0.039676</td>\n",
       "      <td>0.077316</td>\n",
       "      <td>0.866580</td>\n",
       "      <td>0.056111</td>\n",
       "      <td>0.007771</td>\n",
       "      <td>0.303397</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>17</th>\n",
       "      <td>FOTR</td>\n",
       "      <td>Book II</td>\n",
       "      <td>Many Meetings</td>\n",
       "      <td>Chapter 1 . Many Meetings \\n\\nFrodo woke and f...</td>\n",
       "      <td>0.092482</td>\n",
       "      <td>0.047418</td>\n",
       "      <td>0.856683</td>\n",
       "      <td>0.095900</td>\n",
       "      <td>0.073006</td>\n",
       "      <td>0.287620</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>18</th>\n",
       "      <td>FOTR</td>\n",
       "      <td>Book II</td>\n",
       "      <td>The Council of Elrond</td>\n",
       "      <td>Chapter 2 . The Council of Elrond \\n\\n\\n\\nNext...</td>\n",
       "      <td>0.007270</td>\n",
       "      <td>0.074671</td>\n",
       "      <td>0.853501</td>\n",
       "      <td>0.071823</td>\n",
       "      <td>0.040775</td>\n",
       "      <td>0.304679</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>19</th>\n",
       "      <td>FOTR</td>\n",
       "      <td>Book II</td>\n",
       "      <td>The Ring Goes South</td>\n",
       "      <td>Chapter 3 . The Ring Goes South \\n\\n\\n\\nLater ...</td>\n",
       "      <td>0.000085</td>\n",
       "      <td>0.064984</td>\n",
       "      <td>0.863647</td>\n",
       "      <td>0.068535</td>\n",
       "      <td>0.037318</td>\n",
       "      <td>0.313236</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>20</th>\n",
       "      <td>FOTR</td>\n",
       "      <td>Book II</td>\n",
       "      <td>A Journey in the Dark</td>\n",
       "      <td>Chapter 4 . A Journey in the Dark \\n\\n\\n\\nIt w...</td>\n",
       "      <td>-0.018587</td>\n",
       "      <td>0.068742</td>\n",
       "      <td>0.871763</td>\n",
       "      <td>0.059500</td>\n",
       "      <td>0.010430</td>\n",
       "      <td>0.304156</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>21</th>\n",
       "      <td>FOTR</td>\n",
       "      <td>Book II</td>\n",
       "      <td>The Bridge of Khazad-dum</td>\n",
       "      <td>Chapter 5 . The Bridge of Khazad-dum \\n\\n\\n\\nT...</td>\n",
       "      <td>-0.073328</td>\n",
       "      <td>0.097166</td>\n",
       "      <td>0.859562</td>\n",
       "      <td>0.043271</td>\n",
       "      <td>0.014297</td>\n",
       "      <td>0.247021</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>22</th>\n",
       "      <td>FOTR</td>\n",
       "      <td>Book II</td>\n",
       "      <td>Lothlorien</td>\n",
       "      <td>Chapter 6 . Lothlorien \\n\\n\\n\\n'Alas! I Fear w...</td>\n",
       "      <td>0.050481</td>\n",
       "      <td>0.052154</td>\n",
       "      <td>0.882938</td>\n",
       "      <td>0.064901</td>\n",
       "      <td>0.055373</td>\n",
       "      <td>0.290735</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>23</th>\n",
       "      <td>FOTR</td>\n",
       "      <td>Book II</td>\n",
       "      <td>The Mirror of Galadriel</td>\n",
       "      <td>Chapter 7 . The Mirror of Galadriel \\n\\n\\n\\nTh...</td>\n",
       "      <td>0.045263</td>\n",
       "      <td>0.061203</td>\n",
       "      <td>0.860048</td>\n",
       "      <td>0.078742</td>\n",
       "      <td>0.051823</td>\n",
       "      <td>0.331532</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>24</th>\n",
       "      <td>FOTR</td>\n",
       "      <td>Book II</td>\n",
       "      <td>Farewell to Lorien</td>\n",
       "      <td>Chapter 8 . Farewell to Lorien \\n\\n\\n\\nThat ni...</td>\n",
       "      <td>0.091091</td>\n",
       "      <td>0.050251</td>\n",
       "      <td>0.866813</td>\n",
       "      <td>0.082930</td>\n",
       "      <td>0.062147</td>\n",
       "      <td>0.321413</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25</th>\n",
       "      <td>FOTR</td>\n",
       "      <td>Book II</td>\n",
       "      <td>The Great River</td>\n",
       "      <td>Chapter 9 . The Great River \\n\\n\\n\\nFrodo was ...</td>\n",
       "      <td>-0.020113</td>\n",
       "      <td>0.065061</td>\n",
       "      <td>0.890092</td>\n",
       "      <td>0.044852</td>\n",
       "      <td>0.018566</td>\n",
       "      <td>0.306630</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>26</th>\n",
       "      <td>FOTR</td>\n",
       "      <td>Book II</td>\n",
       "      <td>The Breaking of the Fellowship</td>\n",
       "      <td>Chapter 10. The Breaking of the Fellowship \\n\\...</td>\n",
       "      <td>-0.025731</td>\n",
       "      <td>0.099378</td>\n",
       "      <td>0.834948</td>\n",
       "      <td>0.065674</td>\n",
       "      <td>0.003442</td>\n",
       "      <td>0.237485</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   Volume      Book                          Chapters                                               Text  Compound  Negative   Neutral  Positive  Polarity  Subjectivity\n",
       "0    FOTR  Prologue                Concerning Hobbits  1. Concerning Hobbits \\n\\n\\n\\nThis book is lar...  0.133970  0.043674  0.880891  0.075450  0.123098      0.453589\n",
       "1    FOTR  Prologue              Concerning Pipe-weed  2. Concerning Pipe-weed \\n\\n\\n\\nThere is anoth...  0.239865  0.014870  0.903565  0.081609  0.191711      0.472525\n",
       "2    FOTR  Prologue      Of the Ordering of the Shire  3. Of the Ordering of the Shire \\n\\n\\n\\nThe Sh...  0.042571  0.041147  0.903382  0.055471  0.097907      0.441715\n",
       "3    FOTR  Prologue        Of the Finding of the Ring  4. Of the Finding of the Ring \\n\\n\\n\\nAs is to...  0.075828  0.075471  0.841000  0.083485  0.041690      0.336673\n",
       "4    FOTR  Prologue         NOTE ON THE SHIRE RECORDS  NOTE ON THE SHIRE RECORDS \\n\\n\\n\\nAt the end o...  0.115635  0.025703  0.914108  0.060162  0.220372      0.394603\n",
       "5    FOTR    Book I             A Long-expected Party  Chapter 1 . A Long-expected Party \\n\\nWhen Mr....  0.092960  0.050563  0.854325  0.095113  0.063388      0.297571\n",
       "6    FOTR    Book I            The Shadow of the Past  Chapter 2 . The Shadow of the Past \\n\\n\\n\\nThe...  0.023151  0.080364  0.835787  0.083856  0.046675      0.299241\n",
       "7    FOTR    Book I                  Three is Company  Chapter 3 . Three is Company \\n\\n\\n\\n'You ough...  0.062997  0.039382  0.887917  0.072693  0.056428      0.285911\n",
       "8    FOTR    Book I          A Short Cut to Mushrooms  Chapter 4 . A Short Cut to Mushrooms \\n\\n\\n\\nI...  0.011641  0.054406  0.882502  0.063086  0.048184      0.262968\n",
       "9    FOTR    Book I             A Conspiracy Unmasked  Chapter 5 . A Conspiracy Unmasked \\n\\n\\n\\n'Now...  0.083426  0.049687  0.853551  0.096768  0.046885      0.278871\n",
       "10   FOTR    Book I                    The Old Forest  Chapter 6. The Old Forest \\n\\n\\n\\nFrodo woke s...  0.050078  0.059573  0.859174  0.081251  0.011858      0.283457\n",
       "11   FOTR    Book I      In the House of Tom Bombadil  Chapter 7 . In the House of Tom Bombadil \\n\\n\\...  0.109893  0.051551  0.848711  0.099737  0.078520      0.314967\n",
       "12   FOTR    Book I           Fog on the Barrow-Downs  Chapter 8 . Fog on the Barrow-Downs \\n\\n\\n\\nTh...  0.063105  0.056751  0.864729  0.078517  0.012824      0.315016\n",
       "13   FOTR    Book I  At the Sign of The Prancing Pony  Chapter 9 . At the Sign of \\n\\n\\n\\nThe Prancin...  0.063927  0.044007  0.889855  0.066140  0.055907      0.274922\n",
       "14   FOTR    Book I                           Strider  Chapter 10. Strider \\n\\n\\n\\nFrodo, Pippin, and...  0.031716  0.064495  0.854428  0.081069  0.018458      0.252212\n",
       "15   FOTR    Book I               A Knife in the Dark  Chapter 11. A Knife in the Dark \\n\\n\\n\\nAs the... -0.017724  0.078997  0.860830  0.060170  0.012372      0.298401\n",
       "16   FOTR    Book I                Flight to the Ford  Chapter 12 . Flight to the Ford \\n\\n\\n\\nWhen F... -0.039676  0.077316  0.866580  0.056111  0.007771      0.303397\n",
       "17   FOTR   Book II                     Many Meetings  Chapter 1 . Many Meetings \\n\\nFrodo woke and f...  0.092482  0.047418  0.856683  0.095900  0.073006      0.287620\n",
       "18   FOTR   Book II             The Council of Elrond  Chapter 2 . The Council of Elrond \\n\\n\\n\\nNext...  0.007270  0.074671  0.853501  0.071823  0.040775      0.304679\n",
       "19   FOTR   Book II               The Ring Goes South  Chapter 3 . The Ring Goes South \\n\\n\\n\\nLater ...  0.000085  0.064984  0.863647  0.068535  0.037318      0.313236\n",
       "20   FOTR   Book II             A Journey in the Dark  Chapter 4 . A Journey in the Dark \\n\\n\\n\\nIt w... -0.018587  0.068742  0.871763  0.059500  0.010430      0.304156\n",
       "21   FOTR   Book II          The Bridge of Khazad-dum  Chapter 5 . The Bridge of Khazad-dum \\n\\n\\n\\nT... -0.073328  0.097166  0.859562  0.043271  0.014297      0.247021\n",
       "22   FOTR   Book II                        Lothlorien  Chapter 6 . Lothlorien \\n\\n\\n\\n'Alas! I Fear w...  0.050481  0.052154  0.882938  0.064901  0.055373      0.290735\n",
       "23   FOTR   Book II           The Mirror of Galadriel  Chapter 7 . The Mirror of Galadriel \\n\\n\\n\\nTh...  0.045263  0.061203  0.860048  0.078742  0.051823      0.331532\n",
       "24   FOTR   Book II                Farewell to Lorien  Chapter 8 . Farewell to Lorien \\n\\n\\n\\nThat ni...  0.091091  0.050251  0.866813  0.082930  0.062147      0.321413\n",
       "25   FOTR   Book II                   The Great River  Chapter 9 . The Great River \\n\\n\\n\\nFrodo was ... -0.020113  0.065061  0.890092  0.044852  0.018566      0.306630\n",
       "26   FOTR   Book II    The Breaking of the Fellowship  Chapter 10. The Breaking of the Fellowship \\n\\... -0.025731  0.099378  0.834948  0.065674  0.003442      0.237485"
      ]
     },
     "execution_count": 390,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "FOTR_sentiments = pd.merge(FOTR_sentiments, data6, left_index=True, right_index=True)\n",
    "\n",
    "FOTR_sentiments.to_csv(r'C:\\Users\\Nick\\Desktop\\GitProjects\\NLP_projects\\FOTR\\FOTR_sentiments.csv')\n",
    "\n",
    "FOTR_sentiments"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 391,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Total_words</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>19430</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>3355</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>4696</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>8576</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>5279</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>55003</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>59702</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>51900</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>31132</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>27778</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>35355</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11</th>\n",
       "      <td>30042</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12</th>\n",
       "      <td>36168</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13</th>\n",
       "      <td>33940</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14</th>\n",
       "      <td>30897</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15</th>\n",
       "      <td>51077</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16</th>\n",
       "      <td>47165</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>17</th>\n",
       "      <td>48716</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>18</th>\n",
       "      <td>85863</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>19</th>\n",
       "      <td>57180</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>20</th>\n",
       "      <td>61928</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>21</th>\n",
       "      <td>29296</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>22</th>\n",
       "      <td>50428</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>23</th>\n",
       "      <td>36207</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>24</th>\n",
       "      <td>32963</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25</th>\n",
       "      <td>38701</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>26</th>\n",
       "      <td>32624</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "    Total_words\n",
       "0         19430\n",
       "1          3355\n",
       "2          4696\n",
       "3          8576\n",
       "4          5279\n",
       "5         55003\n",
       "6         59702\n",
       "7         51900\n",
       "8         31132\n",
       "9         27778\n",
       "10        35355\n",
       "11        30042\n",
       "12        36168\n",
       "13        33940\n",
       "14        30897\n",
       "15        51077\n",
       "16        47165\n",
       "17        48716\n",
       "18        85863\n",
       "19        57180\n",
       "20        61928\n",
       "21        29296\n",
       "22        50428\n",
       "23        36207\n",
       "24        32963\n",
       "25        38701\n",
       "26        32624"
      ]
     },
     "execution_count": 391,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "chapters = [p1,p2,p3,p4,p5,b1_1,b1_2,b1_3,b1_4,b1_5,b1_6,b1_7,b1_8,b1_9,b1_10,b1_11,b1_12,b2_1,b2_2,b2_3,\n",
    "         b2_4,b2_5,b2_6,b2_7,b2_8,b2_9,b2_10]\n",
    "\n",
    "wordlength_list = list()\n",
    "\n",
    "for chapter in chapters:\n",
    "    count = len(chapter)\n",
    "        \n",
    "    wordlength_list.append(count)  \n",
    "\n",
    "data7 = pd.DataFrame(wordlength_list)  \n",
    "data7=data7.rename(columns={0: 'Total_words'})\n",
    "data7"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 392,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Volume</th>\n",
       "      <th>Book</th>\n",
       "      <th>Chapters</th>\n",
       "      <th>Text</th>\n",
       "      <th>Compound</th>\n",
       "      <th>Negative</th>\n",
       "      <th>Neutral</th>\n",
       "      <th>Positive</th>\n",
       "      <th>Polarity</th>\n",
       "      <th>Subjectivity</th>\n",
       "      <th>Total_words</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>FOTR</td>\n",
       "      <td>Prologue</td>\n",
       "      <td>Concerning Hobbits</td>\n",
       "      <td>1. Concerning Hobbits \\n\\n\\n\\nThis book is lar...</td>\n",
       "      <td>0.133970</td>\n",
       "      <td>0.043674</td>\n",
       "      <td>0.880891</td>\n",
       "      <td>0.075450</td>\n",
       "      <td>0.123098</td>\n",
       "      <td>0.453589</td>\n",
       "      <td>19430</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>FOTR</td>\n",
       "      <td>Prologue</td>\n",
       "      <td>Concerning Pipe-weed</td>\n",
       "      <td>2. Concerning Pipe-weed \\n\\n\\n\\nThere is anoth...</td>\n",
       "      <td>0.239865</td>\n",
       "      <td>0.014870</td>\n",
       "      <td>0.903565</td>\n",
       "      <td>0.081609</td>\n",
       "      <td>0.191711</td>\n",
       "      <td>0.472525</td>\n",
       "      <td>3355</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>FOTR</td>\n",
       "      <td>Prologue</td>\n",
       "      <td>Of the Ordering of the Shire</td>\n",
       "      <td>3. Of the Ordering of the Shire \\n\\n\\n\\nThe Sh...</td>\n",
       "      <td>0.042571</td>\n",
       "      <td>0.041147</td>\n",
       "      <td>0.903382</td>\n",
       "      <td>0.055471</td>\n",
       "      <td>0.097907</td>\n",
       "      <td>0.441715</td>\n",
       "      <td>4696</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>FOTR</td>\n",
       "      <td>Prologue</td>\n",
       "      <td>Of the Finding of the Ring</td>\n",
       "      <td>4. Of the Finding of the Ring \\n\\n\\n\\nAs is to...</td>\n",
       "      <td>0.075828</td>\n",
       "      <td>0.075471</td>\n",
       "      <td>0.841000</td>\n",
       "      <td>0.083485</td>\n",
       "      <td>0.041690</td>\n",
       "      <td>0.336673</td>\n",
       "      <td>8576</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>FOTR</td>\n",
       "      <td>Prologue</td>\n",
       "      <td>NOTE ON THE SHIRE RECORDS</td>\n",
       "      <td>NOTE ON THE SHIRE RECORDS \\n\\n\\n\\nAt the end o...</td>\n",
       "      <td>0.115635</td>\n",
       "      <td>0.025703</td>\n",
       "      <td>0.914108</td>\n",
       "      <td>0.060162</td>\n",
       "      <td>0.220372</td>\n",
       "      <td>0.394603</td>\n",
       "      <td>5279</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "  Volume      Book                      Chapters                                               Text  Compound  Negative   Neutral  Positive  Polarity  Subjectivity  Total_words\n",
       "0   FOTR  Prologue            Concerning Hobbits  1. Concerning Hobbits \\n\\n\\n\\nThis book is lar...  0.133970  0.043674  0.880891  0.075450  0.123098      0.453589        19430\n",
       "1   FOTR  Prologue          Concerning Pipe-weed  2. Concerning Pipe-weed \\n\\n\\n\\nThere is anoth...  0.239865  0.014870  0.903565  0.081609  0.191711      0.472525         3355\n",
       "2   FOTR  Prologue  Of the Ordering of the Shire  3. Of the Ordering of the Shire \\n\\n\\n\\nThe Sh...  0.042571  0.041147  0.903382  0.055471  0.097907      0.441715         4696\n",
       "3   FOTR  Prologue    Of the Finding of the Ring  4. Of the Finding of the Ring \\n\\n\\n\\nAs is to...  0.075828  0.075471  0.841000  0.083485  0.041690      0.336673         8576\n",
       "4   FOTR  Prologue     NOTE ON THE SHIRE RECORDS  NOTE ON THE SHIRE RECORDS \\n\\n\\n\\nAt the end o...  0.115635  0.025703  0.914108  0.060162  0.220372      0.394603         5279"
      ]
     },
     "execution_count": 392,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "FOTR_sentiments = pd.merge(FOTR_sentiments, data7, left_index=True, right_index=True)\n",
    "\n",
    "FOTR_sentiments.to_csv(r'C:\\Users\\Nick\\Desktop\\GitProjects\\NLP_projects\\FOTR\\FOTR_sentiments.csv')\n",
    "\n",
    "FOTR_sentiments.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 393,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Volume</th>\n",
       "      <th>Book</th>\n",
       "      <th>Chapters</th>\n",
       "      <th>Text</th>\n",
       "      <th>Compound</th>\n",
       "      <th>Negative</th>\n",
       "      <th>Neutral</th>\n",
       "      <th>Positive</th>\n",
       "      <th>Polarity</th>\n",
       "      <th>Subjectivity</th>\n",
       "      <th>Total_words</th>\n",
       "      <th>Rating</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>FOTR</td>\n",
       "      <td>Prologue</td>\n",
       "      <td>Concerning Hobbits</td>\n",
       "      <td>1. Concerning Hobbits \\n\\n\\n\\nThis book is lar...</td>\n",
       "      <td>0.133970</td>\n",
       "      <td>0.043674</td>\n",
       "      <td>0.880891</td>\n",
       "      <td>0.075450</td>\n",
       "      <td>0.123098</td>\n",
       "      <td>0.453589</td>\n",
       "      <td>19430</td>\n",
       "      <td>Positive</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>FOTR</td>\n",
       "      <td>Prologue</td>\n",
       "      <td>Concerning Pipe-weed</td>\n",
       "      <td>2. Concerning Pipe-weed \\n\\n\\n\\nThere is anoth...</td>\n",
       "      <td>0.239865</td>\n",
       "      <td>0.014870</td>\n",
       "      <td>0.903565</td>\n",
       "      <td>0.081609</td>\n",
       "      <td>0.191711</td>\n",
       "      <td>0.472525</td>\n",
       "      <td>3355</td>\n",
       "      <td>Positive</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>FOTR</td>\n",
       "      <td>Prologue</td>\n",
       "      <td>Of the Ordering of the Shire</td>\n",
       "      <td>3. Of the Ordering of the Shire \\n\\n\\n\\nThe Sh...</td>\n",
       "      <td>0.042571</td>\n",
       "      <td>0.041147</td>\n",
       "      <td>0.903382</td>\n",
       "      <td>0.055471</td>\n",
       "      <td>0.097907</td>\n",
       "      <td>0.441715</td>\n",
       "      <td>4696</td>\n",
       "      <td>Neutral</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>FOTR</td>\n",
       "      <td>Prologue</td>\n",
       "      <td>Of the Finding of the Ring</td>\n",
       "      <td>4. Of the Finding of the Ring \\n\\n\\n\\nAs is to...</td>\n",
       "      <td>0.075828</td>\n",
       "      <td>0.075471</td>\n",
       "      <td>0.841000</td>\n",
       "      <td>0.083485</td>\n",
       "      <td>0.041690</td>\n",
       "      <td>0.336673</td>\n",
       "      <td>8576</td>\n",
       "      <td>Positive</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>FOTR</td>\n",
       "      <td>Prologue</td>\n",
       "      <td>NOTE ON THE SHIRE RECORDS</td>\n",
       "      <td>NOTE ON THE SHIRE RECORDS \\n\\n\\n\\nAt the end o...</td>\n",
       "      <td>0.115635</td>\n",
       "      <td>0.025703</td>\n",
       "      <td>0.914108</td>\n",
       "      <td>0.060162</td>\n",
       "      <td>0.220372</td>\n",
       "      <td>0.394603</td>\n",
       "      <td>5279</td>\n",
       "      <td>Positive</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>FOTR</td>\n",
       "      <td>Book I</td>\n",
       "      <td>A Long-expected Party</td>\n",
       "      <td>Chapter 1 . A Long-expected Party \\n\\nWhen Mr....</td>\n",
       "      <td>0.092960</td>\n",
       "      <td>0.050563</td>\n",
       "      <td>0.854325</td>\n",
       "      <td>0.095113</td>\n",
       "      <td>0.063388</td>\n",
       "      <td>0.297571</td>\n",
       "      <td>55003</td>\n",
       "      <td>Positive</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>FOTR</td>\n",
       "      <td>Book I</td>\n",
       "      <td>The Shadow of the Past</td>\n",
       "      <td>Chapter 2 . The Shadow of the Past \\n\\n\\n\\nThe...</td>\n",
       "      <td>0.023151</td>\n",
       "      <td>0.080364</td>\n",
       "      <td>0.835787</td>\n",
       "      <td>0.083856</td>\n",
       "      <td>0.046675</td>\n",
       "      <td>0.299241</td>\n",
       "      <td>59702</td>\n",
       "      <td>Neutral</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>FOTR</td>\n",
       "      <td>Book I</td>\n",
       "      <td>Three is Company</td>\n",
       "      <td>Chapter 3 . Three is Company \\n\\n\\n\\n'You ough...</td>\n",
       "      <td>0.062997</td>\n",
       "      <td>0.039382</td>\n",
       "      <td>0.887917</td>\n",
       "      <td>0.072693</td>\n",
       "      <td>0.056428</td>\n",
       "      <td>0.285911</td>\n",
       "      <td>51900</td>\n",
       "      <td>Positive</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>FOTR</td>\n",
       "      <td>Book I</td>\n",
       "      <td>A Short Cut to Mushrooms</td>\n",
       "      <td>Chapter 4 . A Short Cut to Mushrooms \\n\\n\\n\\nI...</td>\n",
       "      <td>0.011641</td>\n",
       "      <td>0.054406</td>\n",
       "      <td>0.882502</td>\n",
       "      <td>0.063086</td>\n",
       "      <td>0.048184</td>\n",
       "      <td>0.262968</td>\n",
       "      <td>31132</td>\n",
       "      <td>Neutral</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>FOTR</td>\n",
       "      <td>Book I</td>\n",
       "      <td>A Conspiracy Unmasked</td>\n",
       "      <td>Chapter 5 . A Conspiracy Unmasked \\n\\n\\n\\n'Now...</td>\n",
       "      <td>0.083426</td>\n",
       "      <td>0.049687</td>\n",
       "      <td>0.853551</td>\n",
       "      <td>0.096768</td>\n",
       "      <td>0.046885</td>\n",
       "      <td>0.278871</td>\n",
       "      <td>27778</td>\n",
       "      <td>Positive</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>FOTR</td>\n",
       "      <td>Book I</td>\n",
       "      <td>The Old Forest</td>\n",
       "      <td>Chapter 6. The Old Forest \\n\\n\\n\\nFrodo woke s...</td>\n",
       "      <td>0.050078</td>\n",
       "      <td>0.059573</td>\n",
       "      <td>0.859174</td>\n",
       "      <td>0.081251</td>\n",
       "      <td>0.011858</td>\n",
       "      <td>0.283457</td>\n",
       "      <td>35355</td>\n",
       "      <td>Positive</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11</th>\n",
       "      <td>FOTR</td>\n",
       "      <td>Book I</td>\n",
       "      <td>In the House of Tom Bombadil</td>\n",
       "      <td>Chapter 7 . In the House of Tom Bombadil \\n\\n\\...</td>\n",
       "      <td>0.109893</td>\n",
       "      <td>0.051551</td>\n",
       "      <td>0.848711</td>\n",
       "      <td>0.099737</td>\n",
       "      <td>0.078520</td>\n",
       "      <td>0.314967</td>\n",
       "      <td>30042</td>\n",
       "      <td>Positive</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12</th>\n",
       "      <td>FOTR</td>\n",
       "      <td>Book I</td>\n",
       "      <td>Fog on the Barrow-Downs</td>\n",
       "      <td>Chapter 8 . Fog on the Barrow-Downs \\n\\n\\n\\nTh...</td>\n",
       "      <td>0.063105</td>\n",
       "      <td>0.056751</td>\n",
       "      <td>0.864729</td>\n",
       "      <td>0.078517</td>\n",
       "      <td>0.012824</td>\n",
       "      <td>0.315016</td>\n",
       "      <td>36168</td>\n",
       "      <td>Positive</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13</th>\n",
       "      <td>FOTR</td>\n",
       "      <td>Book I</td>\n",
       "      <td>At the Sign of The Prancing Pony</td>\n",
       "      <td>Chapter 9 . At the Sign of \\n\\n\\n\\nThe Prancin...</td>\n",
       "      <td>0.063927</td>\n",
       "      <td>0.044007</td>\n",
       "      <td>0.889855</td>\n",
       "      <td>0.066140</td>\n",
       "      <td>0.055907</td>\n",
       "      <td>0.274922</td>\n",
       "      <td>33940</td>\n",
       "      <td>Positive</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14</th>\n",
       "      <td>FOTR</td>\n",
       "      <td>Book I</td>\n",
       "      <td>Strider</td>\n",
       "      <td>Chapter 10. Strider \\n\\n\\n\\nFrodo, Pippin, and...</td>\n",
       "      <td>0.031716</td>\n",
       "      <td>0.064495</td>\n",
       "      <td>0.854428</td>\n",
       "      <td>0.081069</td>\n",
       "      <td>0.018458</td>\n",
       "      <td>0.252212</td>\n",
       "      <td>30897</td>\n",
       "      <td>Neutral</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15</th>\n",
       "      <td>FOTR</td>\n",
       "      <td>Book I</td>\n",
       "      <td>A Knife in the Dark</td>\n",
       "      <td>Chapter 11. A Knife in the Dark \\n\\n\\n\\nAs the...</td>\n",
       "      <td>-0.017724</td>\n",
       "      <td>0.078997</td>\n",
       "      <td>0.860830</td>\n",
       "      <td>0.060170</td>\n",
       "      <td>0.012372</td>\n",
       "      <td>0.298401</td>\n",
       "      <td>51077</td>\n",
       "      <td>Neutral</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16</th>\n",
       "      <td>FOTR</td>\n",
       "      <td>Book I</td>\n",
       "      <td>Flight to the Ford</td>\n",
       "      <td>Chapter 12 . Flight to the Ford \\n\\n\\n\\nWhen F...</td>\n",
       "      <td>-0.039676</td>\n",
       "      <td>0.077316</td>\n",
       "      <td>0.866580</td>\n",
       "      <td>0.056111</td>\n",
       "      <td>0.007771</td>\n",
       "      <td>0.303397</td>\n",
       "      <td>47165</td>\n",
       "      <td>Neutral</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>17</th>\n",
       "      <td>FOTR</td>\n",
       "      <td>Book II</td>\n",
       "      <td>Many Meetings</td>\n",
       "      <td>Chapter 1 . Many Meetings \\n\\nFrodo woke and f...</td>\n",
       "      <td>0.092482</td>\n",
       "      <td>0.047418</td>\n",
       "      <td>0.856683</td>\n",
       "      <td>0.095900</td>\n",
       "      <td>0.073006</td>\n",
       "      <td>0.287620</td>\n",
       "      <td>48716</td>\n",
       "      <td>Positive</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>18</th>\n",
       "      <td>FOTR</td>\n",
       "      <td>Book II</td>\n",
       "      <td>The Council of Elrond</td>\n",
       "      <td>Chapter 2 . The Council of Elrond \\n\\n\\n\\nNext...</td>\n",
       "      <td>0.007270</td>\n",
       "      <td>0.074671</td>\n",
       "      <td>0.853501</td>\n",
       "      <td>0.071823</td>\n",
       "      <td>0.040775</td>\n",
       "      <td>0.304679</td>\n",
       "      <td>85863</td>\n",
       "      <td>Neutral</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>19</th>\n",
       "      <td>FOTR</td>\n",
       "      <td>Book II</td>\n",
       "      <td>The Ring Goes South</td>\n",
       "      <td>Chapter 3 . The Ring Goes South \\n\\n\\n\\nLater ...</td>\n",
       "      <td>0.000085</td>\n",
       "      <td>0.064984</td>\n",
       "      <td>0.863647</td>\n",
       "      <td>0.068535</td>\n",
       "      <td>0.037318</td>\n",
       "      <td>0.313236</td>\n",
       "      <td>57180</td>\n",
       "      <td>Neutral</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>20</th>\n",
       "      <td>FOTR</td>\n",
       "      <td>Book II</td>\n",
       "      <td>A Journey in the Dark</td>\n",
       "      <td>Chapter 4 . A Journey in the Dark \\n\\n\\n\\nIt w...</td>\n",
       "      <td>-0.018587</td>\n",
       "      <td>0.068742</td>\n",
       "      <td>0.871763</td>\n",
       "      <td>0.059500</td>\n",
       "      <td>0.010430</td>\n",
       "      <td>0.304156</td>\n",
       "      <td>61928</td>\n",
       "      <td>Neutral</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>21</th>\n",
       "      <td>FOTR</td>\n",
       "      <td>Book II</td>\n",
       "      <td>The Bridge of Khazad-dum</td>\n",
       "      <td>Chapter 5 . The Bridge of Khazad-dum \\n\\n\\n\\nT...</td>\n",
       "      <td>-0.073328</td>\n",
       "      <td>0.097166</td>\n",
       "      <td>0.859562</td>\n",
       "      <td>0.043271</td>\n",
       "      <td>0.014297</td>\n",
       "      <td>0.247021</td>\n",
       "      <td>29296</td>\n",
       "      <td>Negative</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>22</th>\n",
       "      <td>FOTR</td>\n",
       "      <td>Book II</td>\n",
       "      <td>Lothlorien</td>\n",
       "      <td>Chapter 6 . Lothlorien \\n\\n\\n\\n'Alas! I Fear w...</td>\n",
       "      <td>0.050481</td>\n",
       "      <td>0.052154</td>\n",
       "      <td>0.882938</td>\n",
       "      <td>0.064901</td>\n",
       "      <td>0.055373</td>\n",
       "      <td>0.290735</td>\n",
       "      <td>50428</td>\n",
       "      <td>Positive</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>23</th>\n",
       "      <td>FOTR</td>\n",
       "      <td>Book II</td>\n",
       "      <td>The Mirror of Galadriel</td>\n",
       "      <td>Chapter 7 . The Mirror of Galadriel \\n\\n\\n\\nTh...</td>\n",
       "      <td>0.045263</td>\n",
       "      <td>0.061203</td>\n",
       "      <td>0.860048</td>\n",
       "      <td>0.078742</td>\n",
       "      <td>0.051823</td>\n",
       "      <td>0.331532</td>\n",
       "      <td>36207</td>\n",
       "      <td>Neutral</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>24</th>\n",
       "      <td>FOTR</td>\n",
       "      <td>Book II</td>\n",
       "      <td>Farewell to Lorien</td>\n",
       "      <td>Chapter 8 . Farewell to Lorien \\n\\n\\n\\nThat ni...</td>\n",
       "      <td>0.091091</td>\n",
       "      <td>0.050251</td>\n",
       "      <td>0.866813</td>\n",
       "      <td>0.082930</td>\n",
       "      <td>0.062147</td>\n",
       "      <td>0.321413</td>\n",
       "      <td>32963</td>\n",
       "      <td>Positive</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25</th>\n",
       "      <td>FOTR</td>\n",
       "      <td>Book II</td>\n",
       "      <td>The Great River</td>\n",
       "      <td>Chapter 9 . The Great River \\n\\n\\n\\nFrodo was ...</td>\n",
       "      <td>-0.020113</td>\n",
       "      <td>0.065061</td>\n",
       "      <td>0.890092</td>\n",
       "      <td>0.044852</td>\n",
       "      <td>0.018566</td>\n",
       "      <td>0.306630</td>\n",
       "      <td>38701</td>\n",
       "      <td>Neutral</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>26</th>\n",
       "      <td>FOTR</td>\n",
       "      <td>Book II</td>\n",
       "      <td>The Breaking of the Fellowship</td>\n",
       "      <td>Chapter 10. The Breaking of the Fellowship \\n\\...</td>\n",
       "      <td>-0.025731</td>\n",
       "      <td>0.099378</td>\n",
       "      <td>0.834948</td>\n",
       "      <td>0.065674</td>\n",
       "      <td>0.003442</td>\n",
       "      <td>0.237485</td>\n",
       "      <td>32624</td>\n",
       "      <td>Neutral</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   Volume      Book                          Chapters                                               Text  Compound  Negative   Neutral  Positive  Polarity  Subjectivity  Total_words    Rating\n",
       "0    FOTR  Prologue                Concerning Hobbits  1. Concerning Hobbits \\n\\n\\n\\nThis book is lar...  0.133970  0.043674  0.880891  0.075450  0.123098      0.453589        19430  Positive\n",
       "1    FOTR  Prologue              Concerning Pipe-weed  2. Concerning Pipe-weed \\n\\n\\n\\nThere is anoth...  0.239865  0.014870  0.903565  0.081609  0.191711      0.472525         3355  Positive\n",
       "2    FOTR  Prologue      Of the Ordering of the Shire  3. Of the Ordering of the Shire \\n\\n\\n\\nThe Sh...  0.042571  0.041147  0.903382  0.055471  0.097907      0.441715         4696   Neutral\n",
       "3    FOTR  Prologue        Of the Finding of the Ring  4. Of the Finding of the Ring \\n\\n\\n\\nAs is to...  0.075828  0.075471  0.841000  0.083485  0.041690      0.336673         8576  Positive\n",
       "4    FOTR  Prologue         NOTE ON THE SHIRE RECORDS  NOTE ON THE SHIRE RECORDS \\n\\n\\n\\nAt the end o...  0.115635  0.025703  0.914108  0.060162  0.220372      0.394603         5279  Positive\n",
       "5    FOTR    Book I             A Long-expected Party  Chapter 1 . A Long-expected Party \\n\\nWhen Mr....  0.092960  0.050563  0.854325  0.095113  0.063388      0.297571        55003  Positive\n",
       "6    FOTR    Book I            The Shadow of the Past  Chapter 2 . The Shadow of the Past \\n\\n\\n\\nThe...  0.023151  0.080364  0.835787  0.083856  0.046675      0.299241        59702   Neutral\n",
       "7    FOTR    Book I                  Three is Company  Chapter 3 . Three is Company \\n\\n\\n\\n'You ough...  0.062997  0.039382  0.887917  0.072693  0.056428      0.285911        51900  Positive\n",
       "8    FOTR    Book I          A Short Cut to Mushrooms  Chapter 4 . A Short Cut to Mushrooms \\n\\n\\n\\nI...  0.011641  0.054406  0.882502  0.063086  0.048184      0.262968        31132   Neutral\n",
       "9    FOTR    Book I             A Conspiracy Unmasked  Chapter 5 . A Conspiracy Unmasked \\n\\n\\n\\n'Now...  0.083426  0.049687  0.853551  0.096768  0.046885      0.278871        27778  Positive\n",
       "10   FOTR    Book I                    The Old Forest  Chapter 6. The Old Forest \\n\\n\\n\\nFrodo woke s...  0.050078  0.059573  0.859174  0.081251  0.011858      0.283457        35355  Positive\n",
       "11   FOTR    Book I      In the House of Tom Bombadil  Chapter 7 . In the House of Tom Bombadil \\n\\n\\...  0.109893  0.051551  0.848711  0.099737  0.078520      0.314967        30042  Positive\n",
       "12   FOTR    Book I           Fog on the Barrow-Downs  Chapter 8 . Fog on the Barrow-Downs \\n\\n\\n\\nTh...  0.063105  0.056751  0.864729  0.078517  0.012824      0.315016        36168  Positive\n",
       "13   FOTR    Book I  At the Sign of The Prancing Pony  Chapter 9 . At the Sign of \\n\\n\\n\\nThe Prancin...  0.063927  0.044007  0.889855  0.066140  0.055907      0.274922        33940  Positive\n",
       "14   FOTR    Book I                           Strider  Chapter 10. Strider \\n\\n\\n\\nFrodo, Pippin, and...  0.031716  0.064495  0.854428  0.081069  0.018458      0.252212        30897   Neutral\n",
       "15   FOTR    Book I               A Knife in the Dark  Chapter 11. A Knife in the Dark \\n\\n\\n\\nAs the... -0.017724  0.078997  0.860830  0.060170  0.012372      0.298401        51077   Neutral\n",
       "16   FOTR    Book I                Flight to the Ford  Chapter 12 . Flight to the Ford \\n\\n\\n\\nWhen F... -0.039676  0.077316  0.866580  0.056111  0.007771      0.303397        47165   Neutral\n",
       "17   FOTR   Book II                     Many Meetings  Chapter 1 . Many Meetings \\n\\nFrodo woke and f...  0.092482  0.047418  0.856683  0.095900  0.073006      0.287620        48716  Positive\n",
       "18   FOTR   Book II             The Council of Elrond  Chapter 2 . The Council of Elrond \\n\\n\\n\\nNext...  0.007270  0.074671  0.853501  0.071823  0.040775      0.304679        85863   Neutral\n",
       "19   FOTR   Book II               The Ring Goes South  Chapter 3 . The Ring Goes South \\n\\n\\n\\nLater ...  0.000085  0.064984  0.863647  0.068535  0.037318      0.313236        57180   Neutral\n",
       "20   FOTR   Book II             A Journey in the Dark  Chapter 4 . A Journey in the Dark \\n\\n\\n\\nIt w... -0.018587  0.068742  0.871763  0.059500  0.010430      0.304156        61928   Neutral\n",
       "21   FOTR   Book II          The Bridge of Khazad-dum  Chapter 5 . The Bridge of Khazad-dum \\n\\n\\n\\nT... -0.073328  0.097166  0.859562  0.043271  0.014297      0.247021        29296  Negative\n",
       "22   FOTR   Book II                        Lothlorien  Chapter 6 . Lothlorien \\n\\n\\n\\n'Alas! I Fear w...  0.050481  0.052154  0.882938  0.064901  0.055373      0.290735        50428  Positive\n",
       "23   FOTR   Book II           The Mirror of Galadriel  Chapter 7 . The Mirror of Galadriel \\n\\n\\n\\nTh...  0.045263  0.061203  0.860048  0.078742  0.051823      0.331532        36207   Neutral\n",
       "24   FOTR   Book II                Farewell to Lorien  Chapter 8 . Farewell to Lorien \\n\\n\\n\\nThat ni...  0.091091  0.050251  0.866813  0.082930  0.062147      0.321413        32963  Positive\n",
       "25   FOTR   Book II                   The Great River  Chapter 9 . The Great River \\n\\n\\n\\nFrodo was ... -0.020113  0.065061  0.890092  0.044852  0.018566      0.306630        38701   Neutral\n",
       "26   FOTR   Book II    The Breaking of the Fellowship  Chapter 10. The Breaking of the Fellowship \\n\\... -0.025731  0.099378  0.834948  0.065674  0.003442      0.237485        32624   Neutral"
      ]
     },
     "execution_count": 393,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "def rating(FOTR_sentiments):\n",
    "    if FOTR_sentiments['Compound'] > 0.05:\n",
    "        return 'Positive'\n",
    "    elif FOTR_sentiments['Compound'] < -0.05:\n",
    "        return 'Negative'\n",
    "    else:\n",
    "        return 'Neutral'\n",
    "\n",
    "FOTR_sentiments['Rating'] = FOTR_sentiments.apply(rating, axis=1)\n",
    "\n",
    "FOTR_sentiments.to_csv(r'C:\\Users\\Nick\\Desktop\\GitProjects\\NLP_projects\\FOTR\\FOTR_sentiments.csv')\n",
    "\n",
    "FOTR_sentiments.head(27)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 369,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[[('time', 12),\n",
       "  ('book', 9),\n",
       "  ('hobbit', 8),\n",
       "  ('world', 7),\n",
       "  ('shire', 6),\n",
       "  ('history', 5),\n",
       "  ('peace', 5),\n",
       "  ('folk', 5),\n",
       "  ('sea', 5),\n",
       "  ('age', 4)],\n",
       " [('weed', 4),\n",
       "  ('pipe', 3),\n",
       "  ('shire', 3),\n",
       "  ('claim', 3),\n",
       "  ('leaf', 2),\n",
       "  ('art', 2),\n",
       "  ('family', 2),\n",
       "  ('home', 2),\n",
       "  ('toby', 2),\n",
       "  ('plant', 2)],\n",
       " [('shire', 9),\n",
       "  ('time', 4),\n",
       "  ('family', 3),\n",
       "  ('number', 2),\n",
       "  ('course', 2),\n",
       "  ('norbury', 2),\n",
       "  ('thain', 2),\n",
       "  ('mayor', 2),\n",
       "  ('history', 1),\n",
       "  ('chapter', 1)],\n",
       " [('bilbo', 15),\n",
       "  ('gollum', 12),\n",
       "  ('hobbit', 5),\n",
       "  ('way', 5),\n",
       "  ('island', 5),\n",
       "  ('game', 5),\n",
       "  ('hand', 4),\n",
       "  ('truth', 4),\n",
       "  ('gandalf', 3),\n",
       "  ('treasure', 3)],\n",
       " [('book', 8),\n",
       "  ('shire', 7),\n",
       "  ('history', 6),\n",
       "  ('copy', 5),\n",
       "  ('age', 4),\n",
       "  ('interest', 3),\n",
       "  ('time', 3),\n",
       "  ('war', 3),\n",
       "  ('rivendell', 3),\n",
       "  ('peregrin', 3)],\n",
       " [('bilbo', 55),\n",
       "  ('frodo', 27),\n",
       "  ('end', 24),\n",
       "  ('gandalf', 24),\n",
       "  ('time', 18),\n",
       "  ('party', 15),\n",
       "  ('hobbit', 15),\n",
       "  ('door', 15),\n",
       "  ('course', 14),\n",
       "  ('mr', 11)],\n",
       " [('gandalf', 32),\n",
       "  ('frodo', 32),\n",
       "  ('bilbo', 29),\n",
       "  ('power', 23),\n",
       "  ('gollum', 23),\n",
       "  ('ring', 20),\n",
       "  ('shire', 19),\n",
       "  ('time', 18),\n",
       "  ('way', 16),\n",
       "  ('fire', 13)],\n",
       " [('road', 43),\n",
       "  ('frodo', 37),\n",
       "  ('end', 24),\n",
       "  ('sam', 24),\n",
       "  ('gandalf', 20),\n",
       "  ('way', 20),\n",
       "  ('time', 19),\n",
       "  ('pippin', 19),\n",
       "  ('shire', 16),\n",
       "  ('water', 14)],\n",
       " [('frodo', 25),\n",
       "  ('pippin', 25),\n",
       "  ('farmer', 15),\n",
       "  ('mr', 14),\n",
       "  ('sam', 12),\n",
       "  ('road', 12),\n",
       "  ('time', 12),\n",
       "  ('way', 11),\n",
       "  ('ferry', 11),\n",
       "  ('night', 8)],\n",
       " [('merry', 25),\n",
       "  ('frodo', 22),\n",
       "  ('pippin', 12),\n",
       "  ('water', 11),\n",
       "  ('course', 9),\n",
       "  ('round', 8),\n",
       "  ('shire', 8),\n",
       "  ('way', 8),\n",
       "  ('time', 8),\n",
       "  ('day', 7)],\n",
       " [('merry', 26),\n",
       "  ('path', 21),\n",
       "  ('way', 16),\n",
       "  ('water', 15),\n",
       "  ('side', 14),\n",
       "  ('sam', 11),\n",
       "  ('ground', 11),\n",
       "  ('tree', 11),\n",
       "  ('river', 11),\n",
       "  ('tom', 11)],\n",
       " [('tom', 35),\n",
       "  ('water', 18),\n",
       "  ('house', 14),\n",
       "  ('goldberry', 12),\n",
       "  ('frodo', 12),\n",
       "  ('morning', 12),\n",
       "  ('room', 11),\n",
       "  ('master', 10),\n",
       "  ('bombadil', 9),\n",
       "  ('door', 8)],\n",
       " [('tom', 25),\n",
       "  ('hill', 16),\n",
       "  ('sun', 15),\n",
       "  ('frodo', 14),\n",
       "  ('merry', 13),\n",
       "  ('road', 12),\n",
       "  ('barrow', 11),\n",
       "  ('grass', 11),\n",
       "  ('line', 11),\n",
       "  ('time', 11)],\n",
       " [('frodo', 22),\n",
       "  ('folk', 20),\n",
       "  ('butterbur', 17),\n",
       "  ('man', 15),\n",
       "  ('bree', 13),\n",
       "  ('mr', 13),\n",
       "  ('road', 11),\n",
       "  ('room', 11),\n",
       "  ('moment', 10),\n",
       "  ('shire', 9)],\n",
       " [('strider', 39),\n",
       "  ('frodo', 17),\n",
       "  ('butterbur', 16),\n",
       "  ('name', 14),\n",
       "  ('mr', 14),\n",
       "  ('gandalf', 14),\n",
       "  ('letter', 13),\n",
       "  ('sam', 12),\n",
       "  ('road', 12),\n",
       "  ('night', 11)],\n",
       " [('strider', 43),\n",
       "  ('night', 22),\n",
       "  ('road', 22),\n",
       "  ('hill', 19),\n",
       "  ('day', 16),\n",
       "  ('fire', 16),\n",
       "  ('time', 16),\n",
       "  ('way', 16),\n",
       "  ('frodo', 15),\n",
       "  ('house', 13)],\n",
       " [('road', 38),\n",
       "  ('frodo', 34),\n",
       "  ('strider', 34),\n",
       "  ('horse', 20),\n",
       "  ('way', 19),\n",
       "  ('troll', 15),\n",
       "  ('glorfindel', 15),\n",
       "  ('water', 14),\n",
       "  ('night', 14),\n",
       "  ('ford', 13)],\n",
       " [('frodo', 42),\n",
       "  ('elrond', 25),\n",
       "  ('gandalf', 23),\n",
       "  ('bilbo', 19),\n",
       "  ('time', 15),\n",
       "  ('moment', 12),\n",
       "  ('face', 12),\n",
       "  ('house', 11),\n",
       "  ('music', 11),\n",
       "  ('rivendell', 10)],\n",
       " [('elrond', 37),\n",
       "  ('time', 32),\n",
       "  ('power', 31),\n",
       "  ('gandalf', 29),\n",
       "  ('sauron', 23),\n",
       "  ('day', 22),\n",
       "  ('world', 22),\n",
       "  ('ring', 21),\n",
       "  ('saruman', 20),\n",
       "  ('hand', 18)],\n",
       " [('gandalf', 35),\n",
       "  ('frodo', 24),\n",
       "  ('company', 23),\n",
       "  ('way', 21),\n",
       "  ('night', 17),\n",
       "  ('fire', 17),\n",
       "  ('wind', 16),\n",
       "  ('elrond', 15),\n",
       "  ('boromir', 14),\n",
       "  ('pippin', 13)],\n",
       " [('gandalf', 48),\n",
       "  ('way', 28),\n",
       "  ('water', 27),\n",
       "  ('road', 22),\n",
       "  ('night', 19),\n",
       "  ('company', 19),\n",
       "  ('gimli', 19),\n",
       "  ('stone', 19),\n",
       "  ('nothing', 18),\n",
       "  ('sam', 14)],\n",
       " [('doom', 29),\n",
       "  ('gandalf', 27),\n",
       "  ('door', 20),\n",
       "  ('bridge', 15),\n",
       "  ('fire', 14),\n",
       "  ('way', 12),\n",
       "  ('hall', 12),\n",
       "  ('chamber', 11),\n",
       "  ('side', 11),\n",
       "  ('something', 11)],\n",
       " [('frodo', 28),\n",
       "  ('water', 20),\n",
       "  ('road', 17),\n",
       "  ('haldir', 17),\n",
       "  ('company', 16),\n",
       "  ('gimli', 16),\n",
       "  ('land', 16),\n",
       "  ('wood', 13),\n",
       "  ('world', 13),\n",
       "  ('tree', 13)],\n",
       " [('galadriel', 17),\n",
       "  ('frodo', 14),\n",
       "  ('lady', 12),\n",
       "  ('gandalf', 12),\n",
       "  ('mirror', 11),\n",
       "  ('sam', 11),\n",
       "  ('lord', 10),\n",
       "  ('road', 10),\n",
       "  ('land', 9),\n",
       "  ('cannot', 9)],\n",
       " [('river', 18),\n",
       "  ('company', 14),\n",
       "  ('night', 11),\n",
       "  ('water', 11),\n",
       "  ('lady', 10),\n",
       "  ('land', 9),\n",
       "  ('galadriel', 9),\n",
       "  ('way', 9),\n",
       "  ('boromir', 9),\n",
       "  ('tongue', 9)],\n",
       " [('night', 35),\n",
       "  ('river', 25),\n",
       "  ('water', 23),\n",
       "  ('day', 22),\n",
       "  ('frodo', 17),\n",
       "  ('way', 16),\n",
       "  ('bank', 15),\n",
       "  ('boat', 14),\n",
       "  ('time', 13),\n",
       "  ('company', 11)],\n",
       " [('frodo', 32),\n",
       "  ('sam', 21),\n",
       "  ('boromir', 19),\n",
       "  ('way', 13),\n",
       "  ('river', 10),\n",
       "  ('heart', 9),\n",
       "  ('boat', 9),\n",
       "  ('water', 8),\n",
       "  ('hand', 8),\n",
       "  ('stone', 8)]]"
      ]
     },
     "execution_count": 369,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import io, re, nltk\n",
    "from pathlib import Path\n",
    "from nltk import word_tokenize\n",
    "from nltk.tokenize import RegexpTokenizer\n",
    "from nltk import pos_tag\n",
    "from nltk.corpus import stopwords\n",
    "from nltk import FreqDist\n",
    "\n",
    "chapters = [p1,p2,p3,p4,p5,b1_1,b1_2,b1_3,b1_4,b1_5,b1_6,b1_7,b1_8,b1_9,b1_10,b1_11,b1_12,b2_1,b2_2,b2_3,\n",
    "         b2_4,b2_5,b2_6,b2_7,b2_8,b2_9,b2_10]\n",
    "\n",
    "Noun_freq = list()\n",
    "\n",
    "for chapter in chapters:\n",
    "    chap = chapter\n",
    "    data = chap.replace('\\n', ' ')\n",
    "    data_lower = data.lower()\n",
    "    tokens = RegexpTokenizer(r'\\w+').tokenize(data_lower)\n",
    "    stop = set(stopwords.words('english'))\n",
    "    words = [word for word in tokens if word not in stop]\n",
    "    tagged = pos_tag(words)\n",
    "    nouns = [word for word, pos in tagged if (pos == 'NN')]\n",
    "    Noun1 = (0)\n",
    "            \n",
    "    Noun1 = FreqDist(nouns).most_common(10)\n",
    "    \n",
    "    Noun_freq.append(Noun1)\n",
    "    \n",
    "Noun_freq"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 394,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Lex_density</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0.290210</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0.466338</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0.442961</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>0.335784</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0.377682</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>0.197489</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>0.160103</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>0.174535</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>0.203364</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>0.221496</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>0.215347</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11</th>\n",
       "      <td>0.230714</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12</th>\n",
       "      <td>0.210746</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13</th>\n",
       "      <td>0.222187</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14</th>\n",
       "      <td>0.190234</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15</th>\n",
       "      <td>0.196132</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16</th>\n",
       "      <td>0.186933</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>17</th>\n",
       "      <td>0.192164</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>18</th>\n",
       "      <td>0.133704</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>19</th>\n",
       "      <td>0.180420</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>20</th>\n",
       "      <td>0.175262</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>21</th>\n",
       "      <td>0.220451</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>22</th>\n",
       "      <td>0.173443</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>23</th>\n",
       "      <td>0.195362</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>24</th>\n",
       "      <td>0.218679</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25</th>\n",
       "      <td>0.213239</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>26</th>\n",
       "      <td>0.206858</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "    Lex_density\n",
       "0      0.290210\n",
       "1      0.466338\n",
       "2      0.442961\n",
       "3      0.335784\n",
       "4      0.377682\n",
       "5      0.197489\n",
       "6      0.160103\n",
       "7      0.174535\n",
       "8      0.203364\n",
       "9      0.221496\n",
       "10     0.215347\n",
       "11     0.230714\n",
       "12     0.210746\n",
       "13     0.222187\n",
       "14     0.190234\n",
       "15     0.196132\n",
       "16     0.186933\n",
       "17     0.192164\n",
       "18     0.133704\n",
       "19     0.180420\n",
       "20     0.175262\n",
       "21     0.220451\n",
       "22     0.173443\n",
       "23     0.195362\n",
       "24     0.218679\n",
       "25     0.213239\n",
       "26     0.206858"
      ]
     },
     "execution_count": 394,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# The Lexical Diversity represents the ratio of unique words used to the total number of words in the story.\n",
    "\n",
    "chapters = [p1,p2,p3,p4,p5,b1_1,b1_2,b1_3,b1_4,b1_5,b1_6,b1_7,b1_8,b1_9,b1_10,b1_11,b1_12,b2_1,b2_2,b2_3,\n",
    "         b2_4,b2_5,b2_6,b2_7,b2_8,b2_9,b2_10]\n",
    "\n",
    "def lexical_density(text):\n",
    "    return len(set(text)) / len(text)\n",
    "\n",
    "Lexical_density = list()\n",
    "\n",
    "for chapter in chapters:\n",
    "    chap = chapter\n",
    "    data = chap.replace('\\n', ' ')\n",
    "    data_lower = data.lower()\n",
    "    tokens = RegexpTokenizer(r'\\w+').tokenize(data_lower)\n",
    "    LexDen = (0)\n",
    "            \n",
    "    LexDen = lexical_density(tokens)\n",
    "\n",
    "    Lexical_density.append(LexDen)  \n",
    "\n",
    "LexDen = pd.DataFrame(Lexical_density) \n",
    "LexDen=LexDen.rename(columns={0: 'Lex_density'})\n",
    "LexDen"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 395,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Lex_density_norm</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0.290210</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0.466338</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0.442961</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>0.335784</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0.377682</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>0.197489</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>0.160103</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>0.174535</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>0.203364</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>0.221496</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>0.215347</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11</th>\n",
       "      <td>0.230714</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12</th>\n",
       "      <td>0.210746</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13</th>\n",
       "      <td>0.222187</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14</th>\n",
       "      <td>0.190234</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15</th>\n",
       "      <td>0.196132</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16</th>\n",
       "      <td>0.186933</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>17</th>\n",
       "      <td>0.192164</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>18</th>\n",
       "      <td>0.133704</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>19</th>\n",
       "      <td>0.180420</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>20</th>\n",
       "      <td>0.175262</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>21</th>\n",
       "      <td>0.220451</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>22</th>\n",
       "      <td>0.173443</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>23</th>\n",
       "      <td>0.195362</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>24</th>\n",
       "      <td>0.218679</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25</th>\n",
       "      <td>0.213239</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>26</th>\n",
       "      <td>0.206858</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "    Lex_density_norm\n",
       "0           0.290210\n",
       "1           0.466338\n",
       "2           0.442961\n",
       "3           0.335784\n",
       "4           0.377682\n",
       "5           0.197489\n",
       "6           0.160103\n",
       "7           0.174535\n",
       "8           0.203364\n",
       "9           0.221496\n",
       "10          0.215347\n",
       "11          0.230714\n",
       "12          0.210746\n",
       "13          0.222187\n",
       "14          0.190234\n",
       "15          0.196132\n",
       "16          0.186933\n",
       "17          0.192164\n",
       "18          0.133704\n",
       "19          0.180420\n",
       "20          0.175262\n",
       "21          0.220451\n",
       "22          0.173443\n",
       "23          0.195362\n",
       "24          0.218679\n",
       "25          0.213239\n",
       "26          0.206858"
      ]
     },
     "execution_count": 395,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# The Lexical Diversity represents the ratio of unique words used to the total number of words in the story.\n",
    "\n",
    "chapters = [p1,p2,p3,p4,p5,b1_1,b1_2,b1_3,b1_4,b1_5,b1_6,b1_7,b1_8,b1_9,b1_10,b1_11,b1_12,b2_1,b2_2,b2_3,\n",
    "         b2_4,b2_5,b2_6,b2_7,b2_8,b2_9,b2_10]\n",
    "\n",
    "def lexical_density(text):\n",
    "    return len(set(text)) / len(text)\n",
    "\n",
    "Lexical_density_norm = list()\n",
    "\n",
    "for chapter in chapters:\n",
    "    chap = chapter\n",
    "    data = chap.replace('\\n', ' ')\n",
    "    data_lower = data.lower()\n",
    "    tokens = RegexpTokenizer(r'\\w+').tokenize(data_lower)\n",
    "    tokens = tokens[0:27778]\n",
    "    LexDen1 = (0)\n",
    "            \n",
    "    LexDen1 = lexical_density(tokens)\n",
    "\n",
    "    Lexical_density_norm.append(LexDen1)  \n",
    "\n",
    "LexDen_norm = pd.DataFrame(Lexical_density_norm) \n",
    "LexDen_norm=LexDen_norm.rename(columns={0: 'Lex_density_norm'})\n",
    "LexDen_norm"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 396,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Volume</th>\n",
       "      <th>Book</th>\n",
       "      <th>Chapters</th>\n",
       "      <th>Text</th>\n",
       "      <th>Total_words</th>\n",
       "      <th>Lex_density</th>\n",
       "      <th>Negative</th>\n",
       "      <th>Neutral</th>\n",
       "      <th>Positive</th>\n",
       "      <th>Compound</th>\n",
       "      <th>Rating</th>\n",
       "      <th>Polarity</th>\n",
       "      <th>Subjectivity</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>FOTR</td>\n",
       "      <td>Prologue</td>\n",
       "      <td>Concerning Hobbits</td>\n",
       "      <td>1. Concerning Hobbits \\n\\n\\n\\nThis book is lar...</td>\n",
       "      <td>19430</td>\n",
       "      <td>0.290210</td>\n",
       "      <td>0.043674</td>\n",
       "      <td>0.880891</td>\n",
       "      <td>0.075450</td>\n",
       "      <td>0.133970</td>\n",
       "      <td>Positive</td>\n",
       "      <td>0.123098</td>\n",
       "      <td>0.453589</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>FOTR</td>\n",
       "      <td>Prologue</td>\n",
       "      <td>Concerning Pipe-weed</td>\n",
       "      <td>2. Concerning Pipe-weed \\n\\n\\n\\nThere is anoth...</td>\n",
       "      <td>3355</td>\n",
       "      <td>0.466338</td>\n",
       "      <td>0.014870</td>\n",
       "      <td>0.903565</td>\n",
       "      <td>0.081609</td>\n",
       "      <td>0.239865</td>\n",
       "      <td>Positive</td>\n",
       "      <td>0.191711</td>\n",
       "      <td>0.472525</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>FOTR</td>\n",
       "      <td>Prologue</td>\n",
       "      <td>Of the Ordering of the Shire</td>\n",
       "      <td>3. Of the Ordering of the Shire \\n\\n\\n\\nThe Sh...</td>\n",
       "      <td>4696</td>\n",
       "      <td>0.442961</td>\n",
       "      <td>0.041147</td>\n",
       "      <td>0.903382</td>\n",
       "      <td>0.055471</td>\n",
       "      <td>0.042571</td>\n",
       "      <td>Neutral</td>\n",
       "      <td>0.097907</td>\n",
       "      <td>0.441715</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>FOTR</td>\n",
       "      <td>Prologue</td>\n",
       "      <td>Of the Finding of the Ring</td>\n",
       "      <td>4. Of the Finding of the Ring \\n\\n\\n\\nAs is to...</td>\n",
       "      <td>8576</td>\n",
       "      <td>0.335784</td>\n",
       "      <td>0.075471</td>\n",
       "      <td>0.841000</td>\n",
       "      <td>0.083485</td>\n",
       "      <td>0.075828</td>\n",
       "      <td>Positive</td>\n",
       "      <td>0.041690</td>\n",
       "      <td>0.336673</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>FOTR</td>\n",
       "      <td>Prologue</td>\n",
       "      <td>NOTE ON THE SHIRE RECORDS</td>\n",
       "      <td>NOTE ON THE SHIRE RECORDS \\n\\n\\n\\nAt the end o...</td>\n",
       "      <td>5279</td>\n",
       "      <td>0.377682</td>\n",
       "      <td>0.025703</td>\n",
       "      <td>0.914108</td>\n",
       "      <td>0.060162</td>\n",
       "      <td>0.115635</td>\n",
       "      <td>Positive</td>\n",
       "      <td>0.220372</td>\n",
       "      <td>0.394603</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "  Volume      Book                      Chapters                                               Text  Total_words  Lex_density  Negative   Neutral  Positive  Compound    Rating  Polarity  Subjectivity\n",
       "0   FOTR  Prologue            Concerning Hobbits  1. Concerning Hobbits \\n\\n\\n\\nThis book is lar...        19430     0.290210  0.043674  0.880891  0.075450  0.133970  Positive  0.123098      0.453589\n",
       "1   FOTR  Prologue          Concerning Pipe-weed  2. Concerning Pipe-weed \\n\\n\\n\\nThere is anoth...         3355     0.466338  0.014870  0.903565  0.081609  0.239865  Positive  0.191711      0.472525\n",
       "2   FOTR  Prologue  Of the Ordering of the Shire  3. Of the Ordering of the Shire \\n\\n\\n\\nThe Sh...         4696     0.442961  0.041147  0.903382  0.055471  0.042571   Neutral  0.097907      0.441715\n",
       "3   FOTR  Prologue    Of the Finding of the Ring  4. Of the Finding of the Ring \\n\\n\\n\\nAs is to...         8576     0.335784  0.075471  0.841000  0.083485  0.075828  Positive  0.041690      0.336673\n",
       "4   FOTR  Prologue     NOTE ON THE SHIRE RECORDS  NOTE ON THE SHIRE RECORDS \\n\\n\\n\\nAt the end o...         5279     0.377682  0.025703  0.914108  0.060162  0.115635  Positive  0.220372      0.394603"
      ]
     },
     "execution_count": 396,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "FOTR_sentiments = pd.merge(FOTR_sentiments, LexDen, left_index=True, right_index=True)\n",
    "\n",
    "FOTR_sentiments = FOTR_sentiments[['Volume','Book','Chapters','Text','Total_words','Lex_density','Negative',\n",
    "                                                  'Neutral','Positive','Compound','Rating','Polarity','Subjectivity']]\n",
    "\n",
    "FOTR_sentiments.to_csv(r'C:\\Users\\Nick\\Desktop\\GitProjects\\NLP_projects\\FOTR\\FOTR_sentiments.csv')\n",
    "\n",
    "FOTR_sentiments.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 397,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Volume</th>\n",
       "      <th>Book</th>\n",
       "      <th>Chapters</th>\n",
       "      <th>Text</th>\n",
       "      <th>Total_words</th>\n",
       "      <th>Lex_density</th>\n",
       "      <th>Lex_density_norm</th>\n",
       "      <th>Negative</th>\n",
       "      <th>Neutral</th>\n",
       "      <th>Positive</th>\n",
       "      <th>Compound</th>\n",
       "      <th>Rating</th>\n",
       "      <th>Polarity</th>\n",
       "      <th>Subjectivity</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>FOTR</td>\n",
       "      <td>Prologue</td>\n",
       "      <td>Concerning Hobbits</td>\n",
       "      <td>1. Concerning Hobbits \\n\\n\\n\\nThis book is lar...</td>\n",
       "      <td>19430</td>\n",
       "      <td>0.290210</td>\n",
       "      <td>0.290210</td>\n",
       "      <td>0.043674</td>\n",
       "      <td>0.880891</td>\n",
       "      <td>0.075450</td>\n",
       "      <td>0.133970</td>\n",
       "      <td>Positive</td>\n",
       "      <td>0.123098</td>\n",
       "      <td>0.453589</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>FOTR</td>\n",
       "      <td>Prologue</td>\n",
       "      <td>Concerning Pipe-weed</td>\n",
       "      <td>2. Concerning Pipe-weed \\n\\n\\n\\nThere is anoth...</td>\n",
       "      <td>3355</td>\n",
       "      <td>0.466338</td>\n",
       "      <td>0.466338</td>\n",
       "      <td>0.014870</td>\n",
       "      <td>0.903565</td>\n",
       "      <td>0.081609</td>\n",
       "      <td>0.239865</td>\n",
       "      <td>Positive</td>\n",
       "      <td>0.191711</td>\n",
       "      <td>0.472525</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>FOTR</td>\n",
       "      <td>Prologue</td>\n",
       "      <td>Of the Ordering of the Shire</td>\n",
       "      <td>3. Of the Ordering of the Shire \\n\\n\\n\\nThe Sh...</td>\n",
       "      <td>4696</td>\n",
       "      <td>0.442961</td>\n",
       "      <td>0.442961</td>\n",
       "      <td>0.041147</td>\n",
       "      <td>0.903382</td>\n",
       "      <td>0.055471</td>\n",
       "      <td>0.042571</td>\n",
       "      <td>Neutral</td>\n",
       "      <td>0.097907</td>\n",
       "      <td>0.441715</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>FOTR</td>\n",
       "      <td>Prologue</td>\n",
       "      <td>Of the Finding of the Ring</td>\n",
       "      <td>4. Of the Finding of the Ring \\n\\n\\n\\nAs is to...</td>\n",
       "      <td>8576</td>\n",
       "      <td>0.335784</td>\n",
       "      <td>0.335784</td>\n",
       "      <td>0.075471</td>\n",
       "      <td>0.841000</td>\n",
       "      <td>0.083485</td>\n",
       "      <td>0.075828</td>\n",
       "      <td>Positive</td>\n",
       "      <td>0.041690</td>\n",
       "      <td>0.336673</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>FOTR</td>\n",
       "      <td>Prologue</td>\n",
       "      <td>NOTE ON THE SHIRE RECORDS</td>\n",
       "      <td>NOTE ON THE SHIRE RECORDS \\n\\n\\n\\nAt the end o...</td>\n",
       "      <td>5279</td>\n",
       "      <td>0.377682</td>\n",
       "      <td>0.377682</td>\n",
       "      <td>0.025703</td>\n",
       "      <td>0.914108</td>\n",
       "      <td>0.060162</td>\n",
       "      <td>0.115635</td>\n",
       "      <td>Positive</td>\n",
       "      <td>0.220372</td>\n",
       "      <td>0.394603</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "  Volume      Book                      Chapters                                               Text  Total_words  Lex_density  Lex_density_norm  Negative   Neutral  Positive  Compound    Rating  Polarity  Subjectivity\n",
       "0   FOTR  Prologue            Concerning Hobbits  1. Concerning Hobbits \\n\\n\\n\\nThis book is lar...        19430     0.290210          0.290210  0.043674  0.880891  0.075450  0.133970  Positive  0.123098      0.453589\n",
       "1   FOTR  Prologue          Concerning Pipe-weed  2. Concerning Pipe-weed \\n\\n\\n\\nThere is anoth...         3355     0.466338          0.466338  0.014870  0.903565  0.081609  0.239865  Positive  0.191711      0.472525\n",
       "2   FOTR  Prologue  Of the Ordering of the Shire  3. Of the Ordering of the Shire \\n\\n\\n\\nThe Sh...         4696     0.442961          0.442961  0.041147  0.903382  0.055471  0.042571   Neutral  0.097907      0.441715\n",
       "3   FOTR  Prologue    Of the Finding of the Ring  4. Of the Finding of the Ring \\n\\n\\n\\nAs is to...         8576     0.335784          0.335784  0.075471  0.841000  0.083485  0.075828  Positive  0.041690      0.336673\n",
       "4   FOTR  Prologue     NOTE ON THE SHIRE RECORDS  NOTE ON THE SHIRE RECORDS \\n\\n\\n\\nAt the end o...         5279     0.377682          0.377682  0.025703  0.914108  0.060162  0.115635  Positive  0.220372      0.394603"
      ]
     },
     "execution_count": 397,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "FOTR_sentiments = pd.merge(FOTR_sentiments, LexDen_norm, left_index=True, right_index=True)\n",
    "\n",
    "FOTR_sentiments = FOTR_sentiments[['Volume','Book','Chapters','Text','Total_words','Lex_density','Lex_density_norm','Negative',\n",
    "                                                  'Neutral','Positive','Compound','Rating','Polarity','Subjectivity']]\n",
    "\n",
    "FOTR_sentiments.to_csv(r'C:\\Users\\Nick\\Desktop\\GitProjects\\NLP_projects\\FOTR\\FOTR_sentiments.csv')\n",
    "\n",
    "FOTR_sentiments.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 399,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Volume</th>\n",
       "      <th>Book</th>\n",
       "      <th>Chapters</th>\n",
       "      <th>Text</th>\n",
       "      <th>Total_words</th>\n",
       "      <th>Lex_density</th>\n",
       "      <th>Lex_density_norm</th>\n",
       "      <th>Negative</th>\n",
       "      <th>Neutral</th>\n",
       "      <th>Positive</th>\n",
       "      <th>Compound</th>\n",
       "      <th>Rating</th>\n",
       "      <th>Polarity</th>\n",
       "      <th>Subjectivity</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>FOTR</td>\n",
       "      <td>Prologue</td>\n",
       "      <td>Concerning Hobbits</td>\n",
       "      <td>1. Concerning Hobbits     This book is largely...</td>\n",
       "      <td>19430</td>\n",
       "      <td>0.290210</td>\n",
       "      <td>0.290210</td>\n",
       "      <td>0.043674</td>\n",
       "      <td>0.880891</td>\n",
       "      <td>0.075450</td>\n",
       "      <td>0.133970</td>\n",
       "      <td>Positive</td>\n",
       "      <td>0.123098</td>\n",
       "      <td>0.453589</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>FOTR</td>\n",
       "      <td>Prologue</td>\n",
       "      <td>Concerning Pipe-weed</td>\n",
       "      <td>2. Concerning Pipe-weed     There is another a...</td>\n",
       "      <td>3355</td>\n",
       "      <td>0.466338</td>\n",
       "      <td>0.466338</td>\n",
       "      <td>0.014870</td>\n",
       "      <td>0.903565</td>\n",
       "      <td>0.081609</td>\n",
       "      <td>0.239865</td>\n",
       "      <td>Positive</td>\n",
       "      <td>0.191711</td>\n",
       "      <td>0.472525</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>FOTR</td>\n",
       "      <td>Prologue</td>\n",
       "      <td>Of the Ordering of the Shire</td>\n",
       "      <td>3. Of the Ordering of the Shire     The Shire ...</td>\n",
       "      <td>4696</td>\n",
       "      <td>0.442961</td>\n",
       "      <td>0.442961</td>\n",
       "      <td>0.041147</td>\n",
       "      <td>0.903382</td>\n",
       "      <td>0.055471</td>\n",
       "      <td>0.042571</td>\n",
       "      <td>Neutral</td>\n",
       "      <td>0.097907</td>\n",
       "      <td>0.441715</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>FOTR</td>\n",
       "      <td>Prologue</td>\n",
       "      <td>Of the Finding of the Ring</td>\n",
       "      <td>4. Of the Finding of the Ring     As is told i...</td>\n",
       "      <td>8576</td>\n",
       "      <td>0.335784</td>\n",
       "      <td>0.335784</td>\n",
       "      <td>0.075471</td>\n",
       "      <td>0.841000</td>\n",
       "      <td>0.083485</td>\n",
       "      <td>0.075828</td>\n",
       "      <td>Positive</td>\n",
       "      <td>0.041690</td>\n",
       "      <td>0.336673</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>FOTR</td>\n",
       "      <td>Prologue</td>\n",
       "      <td>NOTE ON THE SHIRE RECORDS</td>\n",
       "      <td>NOTE ON THE SHIRE RECORDS     At the end of th...</td>\n",
       "      <td>5279</td>\n",
       "      <td>0.377682</td>\n",
       "      <td>0.377682</td>\n",
       "      <td>0.025703</td>\n",
       "      <td>0.914108</td>\n",
       "      <td>0.060162</td>\n",
       "      <td>0.115635</td>\n",
       "      <td>Positive</td>\n",
       "      <td>0.220372</td>\n",
       "      <td>0.394603</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "  Volume      Book                      Chapters                                               Text  Total_words  Lex_density  Lex_density_norm  Negative   Neutral  Positive  Compound    Rating  Polarity  Subjectivity\n",
       "0   FOTR  Prologue            Concerning Hobbits  1. Concerning Hobbits     This book is largely...        19430     0.290210          0.290210  0.043674  0.880891  0.075450  0.133970  Positive  0.123098      0.453589\n",
       "1   FOTR  Prologue          Concerning Pipe-weed  2. Concerning Pipe-weed     There is another a...         3355     0.466338          0.466338  0.014870  0.903565  0.081609  0.239865  Positive  0.191711      0.472525\n",
       "2   FOTR  Prologue  Of the Ordering of the Shire  3. Of the Ordering of the Shire     The Shire ...         4696     0.442961          0.442961  0.041147  0.903382  0.055471  0.042571   Neutral  0.097907      0.441715\n",
       "3   FOTR  Prologue    Of the Finding of the Ring  4. Of the Finding of the Ring     As is told i...         8576     0.335784          0.335784  0.075471  0.841000  0.083485  0.075828  Positive  0.041690      0.336673\n",
       "4   FOTR  Prologue     NOTE ON THE SHIRE RECORDS  NOTE ON THE SHIRE RECORDS     At the end of th...         5279     0.377682          0.377682  0.025703  0.914108  0.060162  0.115635  Positive  0.220372      0.394603"
      ]
     },
     "execution_count": 399,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "FOTR_sentiments = FOTR_sentiments.replace('\\n',' ', regex=True) \n",
    "FOTR_sentiments.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 400,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "from nltk import word_tokenize\n",
    "from nltk.stem.snowball import SnowballStemmer\n",
    "from tqdm import tqdm_notebook as tqdm\n",
    "from tqdm import trange\n",
    "\n",
    "\n",
    "def text_emotion(df, column):\n",
    "    '''\n",
    "    INPUT: DataFrame, string\n",
    "    OUTPUT: the original DataFrame with ten new columns for each emotion\n",
    "    '''\n",
    "\n",
    "    new_df = df.copy()\n",
    "\n",
    "    xlsx = pd.read_excel(r'C:\\Users\\Nick\\Desktop\\GitProjects\\NLP_projects\\The_Silmarillion\\NRC-Sentiment-Emotion-Lexicons\\NRC-Sentiment-Emotion-Lexicons\\NRC-Emotion-Lexicon-v0.92\\NRC-Emotion-Lexicon-v0.92-In105Languages-Nov2017Translations.xlsx')\n",
    "    emolex_df = xlsx[['word', 'Positive_NRC','Negative_NRC','Anger', 'Anticipation', 'Disgust', 'Fear','Joy',\n",
    "                      'Sadness', 'Surprise', 'Trust']]\n",
    "    emotions = emolex_df.columns.drop('word')\n",
    "    emo_df = pd.DataFrame(0, index=df.index, columns=emotions)\n",
    "\n",
    "    stemmer = SnowballStemmer(\"english\")\n",
    "\n",
    "    \n",
    "    with tqdm(total=len(list(new_df.iterrows()))) as pbar:\n",
    "        for i, row in new_df.iterrows():\n",
    "            pbar.update(1)\n",
    "            document = word_tokenize(new_df.loc[i][column])\n",
    "            for word in document:\n",
    "                word = stemmer.stem(word.lower())\n",
    "                emo_score = emolex_df[emolex_df.word == word]\n",
    "                if not emo_score.empty:\n",
    "                    for emotion in list(emotions):\n",
    "                        emo_df.at[i, emotion] += emo_score[emotion]\n",
    "\n",
    "    new_df = pd.concat([new_df, emo_df], axis=1)\n",
    "\n",
    "    return new_df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 401,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "c59f96413a244b5bb12f38c00289f29e",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=27), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "FOTR_sentiments_final = text_emotion(FOTR_sentiments, 'Text')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 402,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Volume</th>\n",
       "      <th>Book</th>\n",
       "      <th>Chapters</th>\n",
       "      <th>Text</th>\n",
       "      <th>Total_words</th>\n",
       "      <th>Lex_density</th>\n",
       "      <th>Lex_density_norm</th>\n",
       "      <th>Negative</th>\n",
       "      <th>Neutral</th>\n",
       "      <th>Positive</th>\n",
       "      <th>Compound</th>\n",
       "      <th>Rating</th>\n",
       "      <th>Polarity</th>\n",
       "      <th>Subjectivity</th>\n",
       "      <th>Positive_NRC</th>\n",
       "      <th>Negative_NRC</th>\n",
       "      <th>Anger</th>\n",
       "      <th>Anticipation</th>\n",
       "      <th>Disgust</th>\n",
       "      <th>Fear</th>\n",
       "      <th>Joy</th>\n",
       "      <th>Sadness</th>\n",
       "      <th>Surprise</th>\n",
       "      <th>Trust</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>FOTR</td>\n",
       "      <td>Prologue</td>\n",
       "      <td>Concerning Hobbits</td>\n",
       "      <td>1. Concerning Hobbits     This book is largely...</td>\n",
       "      <td>19430</td>\n",
       "      <td>0.290210</td>\n",
       "      <td>0.290210</td>\n",
       "      <td>0.043674</td>\n",
       "      <td>0.880891</td>\n",
       "      <td>0.075450</td>\n",
       "      <td>0.133970</td>\n",
       "      <td>Positive</td>\n",
       "      <td>0.123098</td>\n",
       "      <td>0.453589</td>\n",
       "      <td>156</td>\n",
       "      <td>63</td>\n",
       "      <td>27</td>\n",
       "      <td>77</td>\n",
       "      <td>10</td>\n",
       "      <td>40</td>\n",
       "      <td>49</td>\n",
       "      <td>39</td>\n",
       "      <td>23</td>\n",
       "      <td>75</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>FOTR</td>\n",
       "      <td>Prologue</td>\n",
       "      <td>Concerning Pipe-weed</td>\n",
       "      <td>2. Concerning Pipe-weed     There is another a...</td>\n",
       "      <td>3355</td>\n",
       "      <td>0.466338</td>\n",
       "      <td>0.466338</td>\n",
       "      <td>0.014870</td>\n",
       "      <td>0.903565</td>\n",
       "      <td>0.081609</td>\n",
       "      <td>0.239865</td>\n",
       "      <td>Positive</td>\n",
       "      <td>0.191711</td>\n",
       "      <td>0.472525</td>\n",
       "      <td>28</td>\n",
       "      <td>10</td>\n",
       "      <td>1</td>\n",
       "      <td>20</td>\n",
       "      <td>1</td>\n",
       "      <td>5</td>\n",
       "      <td>16</td>\n",
       "      <td>8</td>\n",
       "      <td>12</td>\n",
       "      <td>15</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>FOTR</td>\n",
       "      <td>Prologue</td>\n",
       "      <td>Of the Ordering of the Shire</td>\n",
       "      <td>3. Of the Ordering of the Shire     The Shire ...</td>\n",
       "      <td>4696</td>\n",
       "      <td>0.442961</td>\n",
       "      <td>0.442961</td>\n",
       "      <td>0.041147</td>\n",
       "      <td>0.903382</td>\n",
       "      <td>0.055471</td>\n",
       "      <td>0.042571</td>\n",
       "      <td>Neutral</td>\n",
       "      <td>0.097907</td>\n",
       "      <td>0.441715</td>\n",
       "      <td>39</td>\n",
       "      <td>17</td>\n",
       "      <td>4</td>\n",
       "      <td>19</td>\n",
       "      <td>2</td>\n",
       "      <td>10</td>\n",
       "      <td>16</td>\n",
       "      <td>2</td>\n",
       "      <td>3</td>\n",
       "      <td>29</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>FOTR</td>\n",
       "      <td>Prologue</td>\n",
       "      <td>Of the Finding of the Ring</td>\n",
       "      <td>4. Of the Finding of the Ring     As is told i...</td>\n",
       "      <td>8576</td>\n",
       "      <td>0.335784</td>\n",
       "      <td>0.335784</td>\n",
       "      <td>0.075471</td>\n",
       "      <td>0.841000</td>\n",
       "      <td>0.083485</td>\n",
       "      <td>0.075828</td>\n",
       "      <td>Positive</td>\n",
       "      <td>0.041690</td>\n",
       "      <td>0.336673</td>\n",
       "      <td>79</td>\n",
       "      <td>50</td>\n",
       "      <td>22</td>\n",
       "      <td>54</td>\n",
       "      <td>14</td>\n",
       "      <td>32</td>\n",
       "      <td>37</td>\n",
       "      <td>31</td>\n",
       "      <td>41</td>\n",
       "      <td>51</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>FOTR</td>\n",
       "      <td>Prologue</td>\n",
       "      <td>NOTE ON THE SHIRE RECORDS</td>\n",
       "      <td>NOTE ON THE SHIRE RECORDS     At the end of th...</td>\n",
       "      <td>5279</td>\n",
       "      <td>0.377682</td>\n",
       "      <td>0.377682</td>\n",
       "      <td>0.025703</td>\n",
       "      <td>0.914108</td>\n",
       "      <td>0.060162</td>\n",
       "      <td>0.115635</td>\n",
       "      <td>Positive</td>\n",
       "      <td>0.220372</td>\n",
       "      <td>0.394603</td>\n",
       "      <td>36</td>\n",
       "      <td>11</td>\n",
       "      <td>2</td>\n",
       "      <td>9</td>\n",
       "      <td>2</td>\n",
       "      <td>5</td>\n",
       "      <td>6</td>\n",
       "      <td>4</td>\n",
       "      <td>3</td>\n",
       "      <td>19</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "  Volume      Book                      Chapters                                               Text  Total_words  Lex_density  Lex_density_norm  Negative   Neutral  Positive  Compound    Rating  Polarity  Subjectivity  Positive_NRC  Negative_NRC  Anger  Anticipation  Disgust  Fear  Joy  Sadness  Surprise  Trust\n",
       "0   FOTR  Prologue            Concerning Hobbits  1. Concerning Hobbits     This book is largely...        19430     0.290210          0.290210  0.043674  0.880891  0.075450  0.133970  Positive  0.123098      0.453589           156            63     27            77       10    40   49       39        23     75\n",
       "1   FOTR  Prologue          Concerning Pipe-weed  2. Concerning Pipe-weed     There is another a...         3355     0.466338          0.466338  0.014870  0.903565  0.081609  0.239865  Positive  0.191711      0.472525            28            10      1            20        1     5   16        8        12     15\n",
       "2   FOTR  Prologue  Of the Ordering of the Shire  3. Of the Ordering of the Shire     The Shire ...         4696     0.442961          0.442961  0.041147  0.903382  0.055471  0.042571   Neutral  0.097907      0.441715            39            17      4            19        2    10   16        2         3     29\n",
       "3   FOTR  Prologue    Of the Finding of the Ring  4. Of the Finding of the Ring     As is told i...         8576     0.335784          0.335784  0.075471  0.841000  0.083485  0.075828  Positive  0.041690      0.336673            79            50     22            54       14    32   37       31        41     51\n",
       "4   FOTR  Prologue     NOTE ON THE SHIRE RECORDS  NOTE ON THE SHIRE RECORDS     At the end of th...         5279     0.377682          0.377682  0.025703  0.914108  0.060162  0.115635  Positive  0.220372      0.394603            36            11      2             9        2     5    6        4         3     19"
      ]
     },
     "execution_count": 402,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "FOTR_sentiments_final.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 403,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Volume</th>\n",
       "      <th>Book</th>\n",
       "      <th>Chapters</th>\n",
       "      <th>Text</th>\n",
       "      <th>Total_words</th>\n",
       "      <th>Lex_density</th>\n",
       "      <th>Lex_density_norm</th>\n",
       "      <th>Negative</th>\n",
       "      <th>Neutral</th>\n",
       "      <th>Positive</th>\n",
       "      <th>Compound</th>\n",
       "      <th>Rating</th>\n",
       "      <th>Polarity</th>\n",
       "      <th>Subjectivity</th>\n",
       "      <th>Positive_NRC</th>\n",
       "      <th>Negative_NRC</th>\n",
       "      <th>Anger</th>\n",
       "      <th>Anticipation</th>\n",
       "      <th>Disgust</th>\n",
       "      <th>Fear</th>\n",
       "      <th>Joy</th>\n",
       "      <th>Sadness</th>\n",
       "      <th>Surprise</th>\n",
       "      <th>Trust</th>\n",
       "      <th>word_count</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>FOTR</td>\n",
       "      <td>Prologue</td>\n",
       "      <td>Concerning Hobbits</td>\n",
       "      <td>1. Concerning Hobbits     This book is largely...</td>\n",
       "      <td>19430</td>\n",
       "      <td>0.290210</td>\n",
       "      <td>0.290210</td>\n",
       "      <td>0.043674</td>\n",
       "      <td>0.880891</td>\n",
       "      <td>0.075450</td>\n",
       "      <td>0.133970</td>\n",
       "      <td>Positive</td>\n",
       "      <td>0.123098</td>\n",
       "      <td>0.453589</td>\n",
       "      <td>156</td>\n",
       "      <td>63</td>\n",
       "      <td>27</td>\n",
       "      <td>77</td>\n",
       "      <td>10</td>\n",
       "      <td>40</td>\n",
       "      <td>49</td>\n",
       "      <td>39</td>\n",
       "      <td>23</td>\n",
       "      <td>75</td>\n",
       "      <td>3839</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>FOTR</td>\n",
       "      <td>Prologue</td>\n",
       "      <td>Concerning Pipe-weed</td>\n",
       "      <td>2. Concerning Pipe-weed     There is another a...</td>\n",
       "      <td>3355</td>\n",
       "      <td>0.466338</td>\n",
       "      <td>0.466338</td>\n",
       "      <td>0.014870</td>\n",
       "      <td>0.903565</td>\n",
       "      <td>0.081609</td>\n",
       "      <td>0.239865</td>\n",
       "      <td>Positive</td>\n",
       "      <td>0.191711</td>\n",
       "      <td>0.472525</td>\n",
       "      <td>28</td>\n",
       "      <td>10</td>\n",
       "      <td>1</td>\n",
       "      <td>20</td>\n",
       "      <td>1</td>\n",
       "      <td>5</td>\n",
       "      <td>16</td>\n",
       "      <td>8</td>\n",
       "      <td>12</td>\n",
       "      <td>15</td>\n",
       "      <td>680</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>FOTR</td>\n",
       "      <td>Prologue</td>\n",
       "      <td>Of the Ordering of the Shire</td>\n",
       "      <td>3. Of the Ordering of the Shire     The Shire ...</td>\n",
       "      <td>4696</td>\n",
       "      <td>0.442961</td>\n",
       "      <td>0.442961</td>\n",
       "      <td>0.041147</td>\n",
       "      <td>0.903382</td>\n",
       "      <td>0.055471</td>\n",
       "      <td>0.042571</td>\n",
       "      <td>Neutral</td>\n",
       "      <td>0.097907</td>\n",
       "      <td>0.441715</td>\n",
       "      <td>39</td>\n",
       "      <td>17</td>\n",
       "      <td>4</td>\n",
       "      <td>19</td>\n",
       "      <td>2</td>\n",
       "      <td>10</td>\n",
       "      <td>16</td>\n",
       "      <td>2</td>\n",
       "      <td>3</td>\n",
       "      <td>29</td>\n",
       "      <td>947</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>FOTR</td>\n",
       "      <td>Prologue</td>\n",
       "      <td>Of the Finding of the Ring</td>\n",
       "      <td>4. Of the Finding of the Ring     As is told i...</td>\n",
       "      <td>8576</td>\n",
       "      <td>0.335784</td>\n",
       "      <td>0.335784</td>\n",
       "      <td>0.075471</td>\n",
       "      <td>0.841000</td>\n",
       "      <td>0.083485</td>\n",
       "      <td>0.075828</td>\n",
       "      <td>Positive</td>\n",
       "      <td>0.041690</td>\n",
       "      <td>0.336673</td>\n",
       "      <td>79</td>\n",
       "      <td>50</td>\n",
       "      <td>22</td>\n",
       "      <td>54</td>\n",
       "      <td>14</td>\n",
       "      <td>32</td>\n",
       "      <td>37</td>\n",
       "      <td>31</td>\n",
       "      <td>41</td>\n",
       "      <td>51</td>\n",
       "      <td>1854</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>FOTR</td>\n",
       "      <td>Prologue</td>\n",
       "      <td>NOTE ON THE SHIRE RECORDS</td>\n",
       "      <td>NOTE ON THE SHIRE RECORDS     At the end of th...</td>\n",
       "      <td>5279</td>\n",
       "      <td>0.377682</td>\n",
       "      <td>0.377682</td>\n",
       "      <td>0.025703</td>\n",
       "      <td>0.914108</td>\n",
       "      <td>0.060162</td>\n",
       "      <td>0.115635</td>\n",
       "      <td>Positive</td>\n",
       "      <td>0.220372</td>\n",
       "      <td>0.394603</td>\n",
       "      <td>36</td>\n",
       "      <td>11</td>\n",
       "      <td>2</td>\n",
       "      <td>9</td>\n",
       "      <td>2</td>\n",
       "      <td>5</td>\n",
       "      <td>6</td>\n",
       "      <td>4</td>\n",
       "      <td>3</td>\n",
       "      <td>19</td>\n",
       "      <td>1029</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "  Volume      Book                      Chapters                                               Text  Total_words  Lex_density  Lex_density_norm  Negative   Neutral  Positive  Compound    Rating  Polarity  Subjectivity  Positive_NRC  Negative_NRC  Anger  Anticipation  Disgust  Fear  Joy  Sadness  Surprise  Trust  word_count\n",
       "0   FOTR  Prologue            Concerning Hobbits  1. Concerning Hobbits     This book is largely...        19430     0.290210          0.290210  0.043674  0.880891  0.075450  0.133970  Positive  0.123098      0.453589           156            63     27            77       10    40   49       39        23     75        3839\n",
       "1   FOTR  Prologue          Concerning Pipe-weed  2. Concerning Pipe-weed     There is another a...         3355     0.466338          0.466338  0.014870  0.903565  0.081609  0.239865  Positive  0.191711      0.472525            28            10      1            20        1     5   16        8        12     15         680\n",
       "2   FOTR  Prologue  Of the Ordering of the Shire  3. Of the Ordering of the Shire     The Shire ...         4696     0.442961          0.442961  0.041147  0.903382  0.055471  0.042571   Neutral  0.097907      0.441715            39            17      4            19        2    10   16        2         3     29         947\n",
       "3   FOTR  Prologue    Of the Finding of the Ring  4. Of the Finding of the Ring     As is told i...         8576     0.335784          0.335784  0.075471  0.841000  0.083485  0.075828  Positive  0.041690      0.336673            79            50     22            54       14    32   37       31        41     51        1854\n",
       "4   FOTR  Prologue     NOTE ON THE SHIRE RECORDS  NOTE ON THE SHIRE RECORDS     At the end of th...         5279     0.377682          0.377682  0.025703  0.914108  0.060162  0.115635  Positive  0.220372      0.394603            36            11      2             9        2     5    6        4         3     19        1029"
      ]
     },
     "execution_count": 403,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "FOTR_sentiments_final['word_count'] = FOTR_sentiments_final['Text'].apply(tokenize.word_tokenize).apply(len)\n",
    "FOTR_sentiments_final.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 404,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Volume               object\n",
       "Book                 object\n",
       "Chapters             object\n",
       "Text                 object\n",
       "Total_words           int64\n",
       "Lex_density         float64\n",
       "Lex_density_norm    float64\n",
       "Negative            float64\n",
       "Neutral             float64\n",
       "Positive            float64\n",
       "Compound            float64\n",
       "Rating               object\n",
       "Polarity            float64\n",
       "Subjectivity        float64\n",
       "Positive_NRC          int64\n",
       "Negative_NRC          int64\n",
       "Anger                 int64\n",
       "Anticipation          int64\n",
       "Disgust               int64\n",
       "Fear                  int64\n",
       "Joy                   int64\n",
       "Sadness               int64\n",
       "Surprise              int64\n",
       "Trust                 int64\n",
       "word_count            int64\n",
       "dtype: object"
      ]
     },
     "execution_count": 404,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "FOTR_sentiments_final.dtypes"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 405,
   "metadata": {},
   "outputs": [],
   "source": [
    "emotions = ['Positive_NRC','Negative_NRC','Anger','Anticipation','Disgust','Fear','Joy','Sadness','Surprise',\n",
    "                 'Trust']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 406,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Volume</th>\n",
       "      <th>Book</th>\n",
       "      <th>Chapters</th>\n",
       "      <th>Text</th>\n",
       "      <th>Total_words</th>\n",
       "      <th>Lex_density</th>\n",
       "      <th>Lex_density_norm</th>\n",
       "      <th>Negative</th>\n",
       "      <th>Neutral</th>\n",
       "      <th>Positive</th>\n",
       "      <th>Compound</th>\n",
       "      <th>Rating</th>\n",
       "      <th>Polarity</th>\n",
       "      <th>Subjectivity</th>\n",
       "      <th>Positive_NRC</th>\n",
       "      <th>Negative_NRC</th>\n",
       "      <th>Anger</th>\n",
       "      <th>Anticipation</th>\n",
       "      <th>Disgust</th>\n",
       "      <th>Fear</th>\n",
       "      <th>Joy</th>\n",
       "      <th>Sadness</th>\n",
       "      <th>Surprise</th>\n",
       "      <th>Trust</th>\n",
       "      <th>word_count</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>FOTR</td>\n",
       "      <td>Prologue</td>\n",
       "      <td>Concerning Hobbits</td>\n",
       "      <td>1. Concerning Hobbits     This book is largely...</td>\n",
       "      <td>19430</td>\n",
       "      <td>0.290210</td>\n",
       "      <td>0.290210</td>\n",
       "      <td>0.043674</td>\n",
       "      <td>0.880891</td>\n",
       "      <td>0.075450</td>\n",
       "      <td>0.133970</td>\n",
       "      <td>Positive</td>\n",
       "      <td>0.123098</td>\n",
       "      <td>0.453589</td>\n",
       "      <td>0.040636</td>\n",
       "      <td>0.016411</td>\n",
       "      <td>0.007033</td>\n",
       "      <td>0.020057</td>\n",
       "      <td>0.002605</td>\n",
       "      <td>0.010419</td>\n",
       "      <td>0.012764</td>\n",
       "      <td>0.010159</td>\n",
       "      <td>0.005991</td>\n",
       "      <td>0.019536</td>\n",
       "      <td>3839</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>FOTR</td>\n",
       "      <td>Prologue</td>\n",
       "      <td>Concerning Pipe-weed</td>\n",
       "      <td>2. Concerning Pipe-weed     There is another a...</td>\n",
       "      <td>3355</td>\n",
       "      <td>0.466338</td>\n",
       "      <td>0.466338</td>\n",
       "      <td>0.014870</td>\n",
       "      <td>0.903565</td>\n",
       "      <td>0.081609</td>\n",
       "      <td>0.239865</td>\n",
       "      <td>Positive</td>\n",
       "      <td>0.191711</td>\n",
       "      <td>0.472525</td>\n",
       "      <td>0.041176</td>\n",
       "      <td>0.014706</td>\n",
       "      <td>0.001471</td>\n",
       "      <td>0.029412</td>\n",
       "      <td>0.001471</td>\n",
       "      <td>0.007353</td>\n",
       "      <td>0.023529</td>\n",
       "      <td>0.011765</td>\n",
       "      <td>0.017647</td>\n",
       "      <td>0.022059</td>\n",
       "      <td>680</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>FOTR</td>\n",
       "      <td>Prologue</td>\n",
       "      <td>Of the Ordering of the Shire</td>\n",
       "      <td>3. Of the Ordering of the Shire     The Shire ...</td>\n",
       "      <td>4696</td>\n",
       "      <td>0.442961</td>\n",
       "      <td>0.442961</td>\n",
       "      <td>0.041147</td>\n",
       "      <td>0.903382</td>\n",
       "      <td>0.055471</td>\n",
       "      <td>0.042571</td>\n",
       "      <td>Neutral</td>\n",
       "      <td>0.097907</td>\n",
       "      <td>0.441715</td>\n",
       "      <td>0.041183</td>\n",
       "      <td>0.017951</td>\n",
       "      <td>0.004224</td>\n",
       "      <td>0.020063</td>\n",
       "      <td>0.002112</td>\n",
       "      <td>0.010560</td>\n",
       "      <td>0.016895</td>\n",
       "      <td>0.002112</td>\n",
       "      <td>0.003168</td>\n",
       "      <td>0.030623</td>\n",
       "      <td>947</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>FOTR</td>\n",
       "      <td>Prologue</td>\n",
       "      <td>Of the Finding of the Ring</td>\n",
       "      <td>4. Of the Finding of the Ring     As is told i...</td>\n",
       "      <td>8576</td>\n",
       "      <td>0.335784</td>\n",
       "      <td>0.335784</td>\n",
       "      <td>0.075471</td>\n",
       "      <td>0.841000</td>\n",
       "      <td>0.083485</td>\n",
       "      <td>0.075828</td>\n",
       "      <td>Positive</td>\n",
       "      <td>0.041690</td>\n",
       "      <td>0.336673</td>\n",
       "      <td>0.042611</td>\n",
       "      <td>0.026969</td>\n",
       "      <td>0.011866</td>\n",
       "      <td>0.029126</td>\n",
       "      <td>0.007551</td>\n",
       "      <td>0.017260</td>\n",
       "      <td>0.019957</td>\n",
       "      <td>0.016721</td>\n",
       "      <td>0.022114</td>\n",
       "      <td>0.027508</td>\n",
       "      <td>1854</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>FOTR</td>\n",
       "      <td>Prologue</td>\n",
       "      <td>NOTE ON THE SHIRE RECORDS</td>\n",
       "      <td>NOTE ON THE SHIRE RECORDS     At the end of th...</td>\n",
       "      <td>5279</td>\n",
       "      <td>0.377682</td>\n",
       "      <td>0.377682</td>\n",
       "      <td>0.025703</td>\n",
       "      <td>0.914108</td>\n",
       "      <td>0.060162</td>\n",
       "      <td>0.115635</td>\n",
       "      <td>Positive</td>\n",
       "      <td>0.220372</td>\n",
       "      <td>0.394603</td>\n",
       "      <td>0.034985</td>\n",
       "      <td>0.010690</td>\n",
       "      <td>0.001944</td>\n",
       "      <td>0.008746</td>\n",
       "      <td>0.001944</td>\n",
       "      <td>0.004859</td>\n",
       "      <td>0.005831</td>\n",
       "      <td>0.003887</td>\n",
       "      <td>0.002915</td>\n",
       "      <td>0.018465</td>\n",
       "      <td>1029</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "  Volume      Book                      Chapters                                               Text  Total_words  Lex_density  Lex_density_norm  Negative   Neutral  Positive  Compound    Rating  Polarity  Subjectivity  Positive_NRC  Negative_NRC     Anger  Anticipation   Disgust      Fear       Joy   Sadness  Surprise     Trust  word_count\n",
       "0   FOTR  Prologue            Concerning Hobbits  1. Concerning Hobbits     This book is largely...        19430     0.290210          0.290210  0.043674  0.880891  0.075450  0.133970  Positive  0.123098      0.453589      0.040636      0.016411  0.007033      0.020057  0.002605  0.010419  0.012764  0.010159  0.005991  0.019536        3839\n",
       "1   FOTR  Prologue          Concerning Pipe-weed  2. Concerning Pipe-weed     There is another a...         3355     0.466338          0.466338  0.014870  0.903565  0.081609  0.239865  Positive  0.191711      0.472525      0.041176      0.014706  0.001471      0.029412  0.001471  0.007353  0.023529  0.011765  0.017647  0.022059         680\n",
       "2   FOTR  Prologue  Of the Ordering of the Shire  3. Of the Ordering of the Shire     The Shire ...         4696     0.442961          0.442961  0.041147  0.903382  0.055471  0.042571   Neutral  0.097907      0.441715      0.041183      0.017951  0.004224      0.020063  0.002112  0.010560  0.016895  0.002112  0.003168  0.030623         947\n",
       "3   FOTR  Prologue    Of the Finding of the Ring  4. Of the Finding of the Ring     As is told i...         8576     0.335784          0.335784  0.075471  0.841000  0.083485  0.075828  Positive  0.041690      0.336673      0.042611      0.026969  0.011866      0.029126  0.007551  0.017260  0.019957  0.016721  0.022114  0.027508        1854\n",
       "4   FOTR  Prologue     NOTE ON THE SHIRE RECORDS  NOTE ON THE SHIRE RECORDS     At the end of th...         5279     0.377682          0.377682  0.025703  0.914108  0.060162  0.115635  Positive  0.220372      0.394603      0.034985      0.010690  0.001944      0.008746  0.001944  0.004859  0.005831  0.003887  0.002915  0.018465        1029"
      ]
     },
     "execution_count": 406,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "for emotion in emotions:\n",
    "    FOTR_sentiments_final[emotion]=FOTR_sentiments_final[emotion]/FOTR_sentiments_final['word_count']\n",
    "\n",
    "FOTR_sentiments_final.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 407,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Volume</th>\n",
       "      <th>Book</th>\n",
       "      <th>Chapters</th>\n",
       "      <th>Text</th>\n",
       "      <th>Total_words</th>\n",
       "      <th>Lex_density</th>\n",
       "      <th>Lex_density_norm</th>\n",
       "      <th>Negative</th>\n",
       "      <th>Neutral</th>\n",
       "      <th>Positive</th>\n",
       "      <th>Compound</th>\n",
       "      <th>Rating</th>\n",
       "      <th>Polarity</th>\n",
       "      <th>Subjectivity</th>\n",
       "      <th>Positive_NRC</th>\n",
       "      <th>Negative_NRC</th>\n",
       "      <th>Anger</th>\n",
       "      <th>Anticipation</th>\n",
       "      <th>Disgust</th>\n",
       "      <th>Fear</th>\n",
       "      <th>Joy</th>\n",
       "      <th>Sadness</th>\n",
       "      <th>Surprise</th>\n",
       "      <th>Trust</th>\n",
       "      <th>word_count</th>\n",
       "      <th>Rating_num</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>FOTR</td>\n",
       "      <td>Prologue</td>\n",
       "      <td>Concerning Hobbits</td>\n",
       "      <td>1. Concerning Hobbits     This book is largely...</td>\n",
       "      <td>19430</td>\n",
       "      <td>0.290210</td>\n",
       "      <td>0.290210</td>\n",
       "      <td>0.043674</td>\n",
       "      <td>0.880891</td>\n",
       "      <td>0.075450</td>\n",
       "      <td>0.133970</td>\n",
       "      <td>Positive</td>\n",
       "      <td>0.123098</td>\n",
       "      <td>0.453589</td>\n",
       "      <td>0.040636</td>\n",
       "      <td>0.016411</td>\n",
       "      <td>0.007033</td>\n",
       "      <td>0.020057</td>\n",
       "      <td>0.002605</td>\n",
       "      <td>0.010419</td>\n",
       "      <td>0.012764</td>\n",
       "      <td>0.010159</td>\n",
       "      <td>0.005991</td>\n",
       "      <td>0.019536</td>\n",
       "      <td>3839</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>FOTR</td>\n",
       "      <td>Prologue</td>\n",
       "      <td>Concerning Pipe-weed</td>\n",
       "      <td>2. Concerning Pipe-weed     There is another a...</td>\n",
       "      <td>3355</td>\n",
       "      <td>0.466338</td>\n",
       "      <td>0.466338</td>\n",
       "      <td>0.014870</td>\n",
       "      <td>0.903565</td>\n",
       "      <td>0.081609</td>\n",
       "      <td>0.239865</td>\n",
       "      <td>Positive</td>\n",
       "      <td>0.191711</td>\n",
       "      <td>0.472525</td>\n",
       "      <td>0.041176</td>\n",
       "      <td>0.014706</td>\n",
       "      <td>0.001471</td>\n",
       "      <td>0.029412</td>\n",
       "      <td>0.001471</td>\n",
       "      <td>0.007353</td>\n",
       "      <td>0.023529</td>\n",
       "      <td>0.011765</td>\n",
       "      <td>0.017647</td>\n",
       "      <td>0.022059</td>\n",
       "      <td>680</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>FOTR</td>\n",
       "      <td>Prologue</td>\n",
       "      <td>Of the Ordering of the Shire</td>\n",
       "      <td>3. Of the Ordering of the Shire     The Shire ...</td>\n",
       "      <td>4696</td>\n",
       "      <td>0.442961</td>\n",
       "      <td>0.442961</td>\n",
       "      <td>0.041147</td>\n",
       "      <td>0.903382</td>\n",
       "      <td>0.055471</td>\n",
       "      <td>0.042571</td>\n",
       "      <td>Neutral</td>\n",
       "      <td>0.097907</td>\n",
       "      <td>0.441715</td>\n",
       "      <td>0.041183</td>\n",
       "      <td>0.017951</td>\n",
       "      <td>0.004224</td>\n",
       "      <td>0.020063</td>\n",
       "      <td>0.002112</td>\n",
       "      <td>0.010560</td>\n",
       "      <td>0.016895</td>\n",
       "      <td>0.002112</td>\n",
       "      <td>0.003168</td>\n",
       "      <td>0.030623</td>\n",
       "      <td>947</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>FOTR</td>\n",
       "      <td>Prologue</td>\n",
       "      <td>Of the Finding of the Ring</td>\n",
       "      <td>4. Of the Finding of the Ring     As is told i...</td>\n",
       "      <td>8576</td>\n",
       "      <td>0.335784</td>\n",
       "      <td>0.335784</td>\n",
       "      <td>0.075471</td>\n",
       "      <td>0.841000</td>\n",
       "      <td>0.083485</td>\n",
       "      <td>0.075828</td>\n",
       "      <td>Positive</td>\n",
       "      <td>0.041690</td>\n",
       "      <td>0.336673</td>\n",
       "      <td>0.042611</td>\n",
       "      <td>0.026969</td>\n",
       "      <td>0.011866</td>\n",
       "      <td>0.029126</td>\n",
       "      <td>0.007551</td>\n",
       "      <td>0.017260</td>\n",
       "      <td>0.019957</td>\n",
       "      <td>0.016721</td>\n",
       "      <td>0.022114</td>\n",
       "      <td>0.027508</td>\n",
       "      <td>1854</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>FOTR</td>\n",
       "      <td>Prologue</td>\n",
       "      <td>NOTE ON THE SHIRE RECORDS</td>\n",
       "      <td>NOTE ON THE SHIRE RECORDS     At the end of th...</td>\n",
       "      <td>5279</td>\n",
       "      <td>0.377682</td>\n",
       "      <td>0.377682</td>\n",
       "      <td>0.025703</td>\n",
       "      <td>0.914108</td>\n",
       "      <td>0.060162</td>\n",
       "      <td>0.115635</td>\n",
       "      <td>Positive</td>\n",
       "      <td>0.220372</td>\n",
       "      <td>0.394603</td>\n",
       "      <td>0.034985</td>\n",
       "      <td>0.010690</td>\n",
       "      <td>0.001944</td>\n",
       "      <td>0.008746</td>\n",
       "      <td>0.001944</td>\n",
       "      <td>0.004859</td>\n",
       "      <td>0.005831</td>\n",
       "      <td>0.003887</td>\n",
       "      <td>0.002915</td>\n",
       "      <td>0.018465</td>\n",
       "      <td>1029</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>FOTR</td>\n",
       "      <td>Book I</td>\n",
       "      <td>A Long-expected Party</td>\n",
       "      <td>Chapter 1 . A Long-expected Party   When Mr. B...</td>\n",
       "      <td>55003</td>\n",
       "      <td>0.197489</td>\n",
       "      <td>0.197489</td>\n",
       "      <td>0.050563</td>\n",
       "      <td>0.854325</td>\n",
       "      <td>0.095113</td>\n",
       "      <td>0.092960</td>\n",
       "      <td>Positive</td>\n",
       "      <td>0.063388</td>\n",
       "      <td>0.297571</td>\n",
       "      <td>0.035862</td>\n",
       "      <td>0.016163</td>\n",
       "      <td>0.005893</td>\n",
       "      <td>0.022898</td>\n",
       "      <td>0.004377</td>\n",
       "      <td>0.007240</td>\n",
       "      <td>0.020540</td>\n",
       "      <td>0.006735</td>\n",
       "      <td>0.015742</td>\n",
       "      <td>0.020877</td>\n",
       "      <td>11879</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>FOTR</td>\n",
       "      <td>Book I</td>\n",
       "      <td>The Shadow of the Past</td>\n",
       "      <td>Chapter 2 . The Shadow of the Past     The tal...</td>\n",
       "      <td>59702</td>\n",
       "      <td>0.160103</td>\n",
       "      <td>0.160103</td>\n",
       "      <td>0.080364</td>\n",
       "      <td>0.835787</td>\n",
       "      <td>0.083856</td>\n",
       "      <td>0.023151</td>\n",
       "      <td>Neutral</td>\n",
       "      <td>0.046675</td>\n",
       "      <td>0.299241</td>\n",
       "      <td>0.034140</td>\n",
       "      <td>0.025718</td>\n",
       "      <td>0.009926</td>\n",
       "      <td>0.024289</td>\n",
       "      <td>0.006693</td>\n",
       "      <td>0.016769</td>\n",
       "      <td>0.016168</td>\n",
       "      <td>0.014814</td>\n",
       "      <td>0.013160</td>\n",
       "      <td>0.022635</td>\n",
       "      <td>13298</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>FOTR</td>\n",
       "      <td>Book I</td>\n",
       "      <td>Three is Company</td>\n",
       "      <td>Chapter 3 . Three is Company     'You ought to...</td>\n",
       "      <td>51900</td>\n",
       "      <td>0.174535</td>\n",
       "      <td>0.174535</td>\n",
       "      <td>0.039382</td>\n",
       "      <td>0.887917</td>\n",
       "      <td>0.072693</td>\n",
       "      <td>0.062997</td>\n",
       "      <td>Positive</td>\n",
       "      <td>0.056428</td>\n",
       "      <td>0.285911</td>\n",
       "      <td>0.035402</td>\n",
       "      <td>0.020804</td>\n",
       "      <td>0.008129</td>\n",
       "      <td>0.021154</td>\n",
       "      <td>0.005857</td>\n",
       "      <td>0.009703</td>\n",
       "      <td>0.014948</td>\n",
       "      <td>0.010839</td>\n",
       "      <td>0.012325</td>\n",
       "      <td>0.021416</td>\n",
       "      <td>11440</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>FOTR</td>\n",
       "      <td>Book I</td>\n",
       "      <td>A Short Cut to Mushrooms</td>\n",
       "      <td>Chapter 4 . A Short Cut to Mushrooms     In th...</td>\n",
       "      <td>31132</td>\n",
       "      <td>0.203364</td>\n",
       "      <td>0.203364</td>\n",
       "      <td>0.054406</td>\n",
       "      <td>0.882502</td>\n",
       "      <td>0.063086</td>\n",
       "      <td>0.011641</td>\n",
       "      <td>Neutral</td>\n",
       "      <td>0.048184</td>\n",
       "      <td>0.262968</td>\n",
       "      <td>0.031294</td>\n",
       "      <td>0.022760</td>\n",
       "      <td>0.008677</td>\n",
       "      <td>0.016643</td>\n",
       "      <td>0.007966</td>\n",
       "      <td>0.010526</td>\n",
       "      <td>0.014651</td>\n",
       "      <td>0.008962</td>\n",
       "      <td>0.012233</td>\n",
       "      <td>0.019630</td>\n",
       "      <td>7030</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>FOTR</td>\n",
       "      <td>Book I</td>\n",
       "      <td>A Conspiracy Unmasked</td>\n",
       "      <td>Chapter 5 . A Conspiracy Unmasked     'Now we ...</td>\n",
       "      <td>27778</td>\n",
       "      <td>0.221496</td>\n",
       "      <td>0.221496</td>\n",
       "      <td>0.049687</td>\n",
       "      <td>0.853551</td>\n",
       "      <td>0.096768</td>\n",
       "      <td>0.083426</td>\n",
       "      <td>Positive</td>\n",
       "      <td>0.046885</td>\n",
       "      <td>0.278871</td>\n",
       "      <td>0.032920</td>\n",
       "      <td>0.019068</td>\n",
       "      <td>0.005541</td>\n",
       "      <td>0.021675</td>\n",
       "      <td>0.003911</td>\n",
       "      <td>0.008963</td>\n",
       "      <td>0.014016</td>\n",
       "      <td>0.009615</td>\n",
       "      <td>0.014016</td>\n",
       "      <td>0.019883</td>\n",
       "      <td>6136</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "  Volume      Book                      Chapters                                               Text  Total_words  Lex_density  Lex_density_norm  Negative   Neutral  Positive  Compound    Rating  Polarity  Subjectivity  Positive_NRC  Negative_NRC     Anger  Anticipation   Disgust      Fear       Joy   Sadness  Surprise     Trust  word_count  Rating_num\n",
       "0   FOTR  Prologue            Concerning Hobbits  1. Concerning Hobbits     This book is largely...        19430     0.290210          0.290210  0.043674  0.880891  0.075450  0.133970  Positive  0.123098      0.453589      0.040636      0.016411  0.007033      0.020057  0.002605  0.010419  0.012764  0.010159  0.005991  0.019536        3839           1\n",
       "1   FOTR  Prologue          Concerning Pipe-weed  2. Concerning Pipe-weed     There is another a...         3355     0.466338          0.466338  0.014870  0.903565  0.081609  0.239865  Positive  0.191711      0.472525      0.041176      0.014706  0.001471      0.029412  0.001471  0.007353  0.023529  0.011765  0.017647  0.022059         680           1\n",
       "2   FOTR  Prologue  Of the Ordering of the Shire  3. Of the Ordering of the Shire     The Shire ...         4696     0.442961          0.442961  0.041147  0.903382  0.055471  0.042571   Neutral  0.097907      0.441715      0.041183      0.017951  0.004224      0.020063  0.002112  0.010560  0.016895  0.002112  0.003168  0.030623         947           0\n",
       "3   FOTR  Prologue    Of the Finding of the Ring  4. Of the Finding of the Ring     As is told i...         8576     0.335784          0.335784  0.075471  0.841000  0.083485  0.075828  Positive  0.041690      0.336673      0.042611      0.026969  0.011866      0.029126  0.007551  0.017260  0.019957  0.016721  0.022114  0.027508        1854           1\n",
       "4   FOTR  Prologue     NOTE ON THE SHIRE RECORDS  NOTE ON THE SHIRE RECORDS     At the end of th...         5279     0.377682          0.377682  0.025703  0.914108  0.060162  0.115635  Positive  0.220372      0.394603      0.034985      0.010690  0.001944      0.008746  0.001944  0.004859  0.005831  0.003887  0.002915  0.018465        1029           1\n",
       "5   FOTR    Book I         A Long-expected Party  Chapter 1 . A Long-expected Party   When Mr. B...        55003     0.197489          0.197489  0.050563  0.854325  0.095113  0.092960  Positive  0.063388      0.297571      0.035862      0.016163  0.005893      0.022898  0.004377  0.007240  0.020540  0.006735  0.015742  0.020877       11879           1\n",
       "6   FOTR    Book I        The Shadow of the Past  Chapter 2 . The Shadow of the Past     The tal...        59702     0.160103          0.160103  0.080364  0.835787  0.083856  0.023151   Neutral  0.046675      0.299241      0.034140      0.025718  0.009926      0.024289  0.006693  0.016769  0.016168  0.014814  0.013160  0.022635       13298           0\n",
       "7   FOTR    Book I              Three is Company  Chapter 3 . Three is Company     'You ought to...        51900     0.174535          0.174535  0.039382  0.887917  0.072693  0.062997  Positive  0.056428      0.285911      0.035402      0.020804  0.008129      0.021154  0.005857  0.009703  0.014948  0.010839  0.012325  0.021416       11440           1\n",
       "8   FOTR    Book I      A Short Cut to Mushrooms  Chapter 4 . A Short Cut to Mushrooms     In th...        31132     0.203364          0.203364  0.054406  0.882502  0.063086  0.011641   Neutral  0.048184      0.262968      0.031294      0.022760  0.008677      0.016643  0.007966  0.010526  0.014651  0.008962  0.012233  0.019630        7030           0\n",
       "9   FOTR    Book I         A Conspiracy Unmasked  Chapter 5 . A Conspiracy Unmasked     'Now we ...        27778     0.221496          0.221496  0.049687  0.853551  0.096768  0.083426  Positive  0.046885      0.278871      0.032920      0.019068  0.005541      0.021675  0.003911  0.008963  0.014016  0.009615  0.014016  0.019883        6136           1"
      ]
     },
     "execution_count": 407,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "def ratings(FOTR_sentiments_final):\n",
    "    if FOTR_sentiments_final['Compound'] > 0.05:\n",
    "        return 1\n",
    "    elif FOTR_sentiments_final['Compound'] < -0.05:\n",
    "        return -1\n",
    "    else:\n",
    "        return 0\n",
    "\n",
    "FOTR_sentiments_final['Rating_num'] = FOTR_sentiments_final.apply(ratings, axis=1)\n",
    "\n",
    "FOTR_sentiments_final.to_csv(r'C:\\Users\\Nick\\Desktop\\GitProjects\\NLP_projects\\FOTR\\FOTR_sentiments.csv')\n",
    "\n",
    "FOTR_sentiments_final.head(10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}