preprocessing.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "accessing previously extracted data from scraping"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "import json\n",
    "\n",
    "with open(\"data/article_metadata/science.json\", 'r') as file:\n",
    "    science_articles = json.loads(file.read())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "with open(\"data/article_metadata/conspiracy.json\", 'r') as file:\n",
    "    conspiracy_articles = json.loads(file.read())"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "combining both kinds of articles and converting the combination to a DataFrame"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "# creating a pandas dataframe from the two lists for the two types of articles\n",
    "df_science = pd.DataFrame.from_dict(science_articles)\n",
    "df_conspiracy = pd.DataFrame.from_dict(conspiracy_articles)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "# creating a new column to distinguish scientific articles from conspiracy ones\n",
    "df_science['article_type'] = \"science\"\n",
    "df_conspiracy['article_type'] = \"conspiracy\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 64,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(728, 3)"
      ]
     },
     "execution_count": 64,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_science.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 65,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(1187, 3)"
      ]
     },
     "execution_count": 65,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_conspiracy.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>title</th>\n",
       "      <th>link</th>\n",
       "      <th>article_type</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Sound Environment during Dental Treatment in R...</td>\n",
       "      <td>https://www.mdpi.com/journal/acoustics/2624-59...</td>\n",
       "      <td>science</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Factors Associated with the Prevalence and Tre...</td>\n",
       "      <td>https://www.mdpi.com/journal/adolescents/2673-...</td>\n",
       "      <td>science</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>The Impact of Comprehensive Rehabilitation on ...</td>\n",
       "      <td>https://www.mdpi.com/journal/arm/2543-6031/91/...</td>\n",
       "      <td>science</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>COVID-19 Acute Respiratory Distress Syndrome: ...</td>\n",
       "      <td>https://www.mdpi.com/journal/arm/2543-6031/91/...</td>\n",
       "      <td>science</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>The Clinical Significance of Aspergillus Detec...</td>\n",
       "      <td>https://www.mdpi.com/journal/arm/2543-6031/91/...</td>\n",
       "      <td>science</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                               title  \\\n",
       "0  Sound Environment during Dental Treatment in R...   \n",
       "1  Factors Associated with the Prevalence and Tre...   \n",
       "2  The Impact of Comprehensive Rehabilitation on ...   \n",
       "3  COVID-19 Acute Respiratory Distress Syndrome: ...   \n",
       "4  The Clinical Significance of Aspergillus Detec...   \n",
       "\n",
       "                                                link article_type  \n",
       "0  https://www.mdpi.com/journal/acoustics/2624-59...      science  \n",
       "1  https://www.mdpi.com/journal/adolescents/2673-...      science  \n",
       "2  https://www.mdpi.com/journal/arm/2543-6031/91/...      science  \n",
       "3  https://www.mdpi.com/journal/arm/2543-6031/91/...      science  \n",
       "4  https://www.mdpi.com/journal/arm/2543-6031/91/...      science  "
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_science.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>title</th>\n",
       "      <th>link</th>\n",
       "      <th>article_type</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Italian Health Minister Under Investigation fo...</td>\n",
       "      <td>https://www.abovetopsecret.com/forum/thread134...</td>\n",
       "      <td>conspiracy</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>COVID vaccination rates \\xe2\\x80\\x98alarmingly...</td>\n",
       "      <td>https://www.abovetopsecret.com/forum/thread134...</td>\n",
       "      <td>conspiracy</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>25% of COVID Vaxxed Now Have VAIDS, Cambridge ...</td>\n",
       "      <td>https://www.abovetopsecret.com/forum/thread134...</td>\n",
       "      <td>conspiracy</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>54% of US Youth are Chronically Ill* America's...</td>\n",
       "      <td>https://childrenshealthdefense.org/follow-the-...</td>\n",
       "      <td>conspiracy</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>lobbied for COVID-19</td>\n",
       "      <td>https://www.leefang.com/p/pfizer-quietly-finan...</td>\n",
       "      <td>conspiracy</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                               title  \\\n",
       "0  Italian Health Minister Under Investigation fo...   \n",
       "1  COVID vaccination rates \\xe2\\x80\\x98alarmingly...   \n",
       "2  25% of COVID Vaxxed Now Have VAIDS, Cambridge ...   \n",
       "3  54% of US Youth are Chronically Ill* America's...   \n",
       "4                               lobbied for COVID-19   \n",
       "\n",
       "                                                link article_type  \n",
       "0  https://www.abovetopsecret.com/forum/thread134...   conspiracy  \n",
       "1  https://www.abovetopsecret.com/forum/thread134...   conspiracy  \n",
       "2  https://www.abovetopsecret.com/forum/thread134...   conspiracy  \n",
       "3  https://childrenshealthdefense.org/follow-the-...   conspiracy  \n",
       "4  https://www.leefang.com/p/pfizer-quietly-finan...   conspiracy  "
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_conspiracy.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "# merging the two dataframes whilst also preserving order\n",
    "df = pd.concat([df_science, df_conspiracy], axis=0, ignore_index=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>title</th>\n",
       "      <th>link</th>\n",
       "      <th>article_type</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Sound Environment during Dental Treatment in R...</td>\n",
       "      <td>https://www.mdpi.com/journal/acoustics/2624-59...</td>\n",
       "      <td>science</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Factors Associated with the Prevalence and Tre...</td>\n",
       "      <td>https://www.mdpi.com/journal/adolescents/2673-...</td>\n",
       "      <td>science</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>The Impact of Comprehensive Rehabilitation on ...</td>\n",
       "      <td>https://www.mdpi.com/journal/arm/2543-6031/91/...</td>\n",
       "      <td>science</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>COVID-19 Acute Respiratory Distress Syndrome: ...</td>\n",
       "      <td>https://www.mdpi.com/journal/arm/2543-6031/91/...</td>\n",
       "      <td>science</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>The Clinical Significance of Aspergillus Detec...</td>\n",
       "      <td>https://www.mdpi.com/journal/arm/2543-6031/91/...</td>\n",
       "      <td>science</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                               title  \\\n",
       "0  Sound Environment during Dental Treatment in R...   \n",
       "1  Factors Associated with the Prevalence and Tre...   \n",
       "2  The Impact of Comprehensive Rehabilitation on ...   \n",
       "3  COVID-19 Acute Respiratory Distress Syndrome: ...   \n",
       "4  The Clinical Significance of Aspergillus Detec...   \n",
       "\n",
       "                                                link article_type  \n",
       "0  https://www.mdpi.com/journal/acoustics/2624-59...      science  \n",
       "1  https://www.mdpi.com/journal/adolescents/2673-...      science  \n",
       "2  https://www.mdpi.com/journal/arm/2543-6031/91/...      science  \n",
       "3  https://www.mdpi.com/journal/arm/2543-6031/91/...      science  \n",
       "4  https://www.mdpi.com/journal/arm/2543-6031/91/...      science  "
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "scraping to obtain the textual content in all short-listed articles"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/isobarbaric/Library/Caches/pypoetry/virtualenvs/covid19-classifier-EHzMhhWe-py3.9/lib/python3.9/site-packages/numpy/core/fromnumeric.py:59: FutureWarning: 'DataFrame.swapaxes' is deprecated and will be removed in a future version. Please use 'DataFrame.transpose' instead.\n",
      "  return bound(*args, **kwds)\n"
     ]
    }
   ],
   "source": [
    "# splitting the df into 30 different parts to improve async scraping efficiency\n",
    "split_df = np.array_split(df, 30)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>title</th>\n",
       "      <th>link</th>\n",
       "      <th>article_type</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Sound Environment during Dental Treatment in R...</td>\n",
       "      <td>https://www.mdpi.com/journal/acoustics/2624-59...</td>\n",
       "      <td>science</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Factors Associated with the Prevalence and Tre...</td>\n",
       "      <td>https://www.mdpi.com/journal/adolescents/2673-...</td>\n",
       "      <td>science</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>The Impact of Comprehensive Rehabilitation on ...</td>\n",
       "      <td>https://www.mdpi.com/journal/arm/2543-6031/91/...</td>\n",
       "      <td>science</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>COVID-19 Acute Respiratory Distress Syndrome: ...</td>\n",
       "      <td>https://www.mdpi.com/journal/arm/2543-6031/91/...</td>\n",
       "      <td>science</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>The Clinical Significance of Aspergillus Detec...</td>\n",
       "      <td>https://www.mdpi.com/journal/arm/2543-6031/91/...</td>\n",
       "      <td>science</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>59</th>\n",
       "      <td>Coronavirus / COVID-19</td>\n",
       "      <td>https://www.everydayhealth.com/coronavirus/</td>\n",
       "      <td>science</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>60</th>\n",
       "      <td>Original Research \\n Accepted on 22 Dec 2023\\n...</td>\n",
       "      <td>https://www.frontiersin.org/articles/10.3389/f...</td>\n",
       "      <td>science</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>61</th>\n",
       "      <td>Original Research \\n Accepted on 22 Dec 2023\\n...</td>\n",
       "      <td>https://www.frontiersin.org/articles/10.3389/f...</td>\n",
       "      <td>science</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>62</th>\n",
       "      <td>Original Research \\n Published on 22 Dec 2023\\...</td>\n",
       "      <td>https://www.frontiersin.org/articles/10.3389/f...</td>\n",
       "      <td>science</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>63</th>\n",
       "      <td>Probiotics delay COVID infection and cut symptoms</td>\n",
       "      <td>https://www.futurity.org/probiotics-covid-infe...</td>\n",
       "      <td>science</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>64 rows × 3 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                title  \\\n",
       "0   Sound Environment during Dental Treatment in R...   \n",
       "1   Factors Associated with the Prevalence and Tre...   \n",
       "2   The Impact of Comprehensive Rehabilitation on ...   \n",
       "3   COVID-19 Acute Respiratory Distress Syndrome: ...   \n",
       "4   The Clinical Significance of Aspergillus Detec...   \n",
       "..                                                ...   \n",
       "59                             Coronavirus / COVID-19   \n",
       "60  Original Research \\n Accepted on 22 Dec 2023\\n...   \n",
       "61  Original Research \\n Accepted on 22 Dec 2023\\n...   \n",
       "62  Original Research \\n Published on 22 Dec 2023\\...   \n",
       "63  Probiotics delay COVID infection and cut symptoms   \n",
       "\n",
       "                                                 link article_type  \n",
       "0   https://www.mdpi.com/journal/acoustics/2624-59...      science  \n",
       "1   https://www.mdpi.com/journal/adolescents/2673-...      science  \n",
       "2   https://www.mdpi.com/journal/arm/2543-6031/91/...      science  \n",
       "3   https://www.mdpi.com/journal/arm/2543-6031/91/...      science  \n",
       "4   https://www.mdpi.com/journal/arm/2543-6031/91/...      science  \n",
       "..                                                ...          ...  \n",
       "59        https://www.everydayhealth.com/coronavirus/      science  \n",
       "60  https://www.frontiersin.org/articles/10.3389/f...      science  \n",
       "61  https://www.frontiersin.org/articles/10.3389/f...      science  \n",
       "62  https://www.frontiersin.org/articles/10.3389/f...      science  \n",
       "63  https://www.futurity.org/probiotics-covid-infe...      science  \n",
       "\n",
       "[64 rows x 3 columns]"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "split_df[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "from newspaper import Article\n",
    "import time\n",
    "\n",
    "# obtaining the texts for the articles in current chunk\n",
    "def parse_chunk(articles: list) -> list:\n",
    "    extracted_text = []\n",
    "\n",
    "    for article in articles:\n",
    "        try:\n",
    "            # attempting to obtain the article text for the current article\n",
    "            current_article = Article(article[1])\n",
    "            current_article.download(), current_article.parse()\n",
    "\n",
    "            extracted_text.append(current_article.text)\n",
    "        except Exception:\n",
    "            # adding in a placeholder value to storage to indicate failure\n",
    "            extracted_text.append(\"N/A\")\n",
    "\n",
    "    return extracted_text"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "30"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(split_df)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "# creating a variable to store all the text for all of the articles\n",
    "all_text = []"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "---- Chunk #1 ----\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "55.005823850631714 seconds elapsed\n",
      "---- Chunk #2 ----\n",
      "76.12486505508423 seconds elapsed\n",
      "---- Chunk #3 ----\n",
      "82.7843759059906 seconds elapsed\n",
      "---- Chunk #4 ----\n",
      "65.37595224380493 seconds elapsed\n",
      "---- Chunk #5 ----\n",
      "99.77528190612793 seconds elapsed\n",
      "---- Chunk #6 ----\n",
      "100.16847610473633 seconds elapsed\n",
      "---- Chunk #7 ----\n",
      "76.275141954422 seconds elapsed\n",
      "---- Chunk #8 ----\n",
      "70.33864212036133 seconds elapsed\n",
      "---- Chunk #9 ----\n",
      "76.3492739200592 seconds elapsed\n",
      "---- Chunk #10 ----\n",
      "74.95590567588806 seconds elapsed\n",
      "---- Chunk #11 ----\n",
      "77.28997826576233 seconds elapsed\n",
      "---- Chunk #12 ----\n",
      "59.91050481796265 seconds elapsed\n",
      "---- Chunk #13 ----\n",
      "37.14605498313904 seconds elapsed\n",
      "---- Chunk #14 ----\n",
      "27.06571388244629 seconds elapsed\n",
      "---- Chunk #15 ----\n",
      "77.11118698120117 seconds elapsed\n",
      "---- Chunk #16 ----\n",
      "152.51395416259766 seconds elapsed\n",
      "---- Chunk #17 ----\n",
      "166.90852093696594 seconds elapsed\n",
      "---- Chunk #18 ----\n",
      "29.693684339523315 seconds elapsed\n",
      "---- Chunk #19 ----\n",
      "29.037486791610718 seconds elapsed\n",
      "---- Chunk #20 ----\n",
      "44.81970500946045 seconds elapsed\n",
      "---- Chunk #21 ----\n",
      "47.835012912750244 seconds elapsed\n",
      "---- Chunk #22 ----\n",
      "81.07963418960571 seconds elapsed\n",
      "---- Chunk #23 ----\n",
      "44.23537516593933 seconds elapsed\n",
      "---- Chunk #24 ----\n",
      "71.1778519153595 seconds elapsed\n",
      "---- Chunk #25 ----\n",
      "45.66049289703369 seconds elapsed\n",
      "---- Chunk #26 ----\n",
      "31.908576726913452 seconds elapsed\n",
      "---- Chunk #27 ----\n",
      "83.80009889602661 seconds elapsed\n",
      "---- Chunk #28 ----\n",
      "30.258076190948486 seconds elapsed\n",
      "---- Chunk #29 ----\n",
      "76.77220106124878 seconds elapsed\n",
      "---- Chunk #30 ----\n",
      "94.3424232006073 seconds elapsed\n"
     ]
    }
   ],
   "source": [
    "# loop over\n",
    "for index, chunk in enumerate(split_df):\n",
    "    # printing out a status message - specifying the current chunk number\n",
    "    print(f'---- Chunk #{index+1} ----')\n",
    "\n",
    "    # creating a variable to store the time execution is started\n",
    "    start = time.time()\n",
    "\n",
    "    # parsing the current chunk\n",
    "    parsed_current_chunk = parse_chunk(chunk.values)\n",
    "\n",
    "    # creating a variable to store the time execution is ended\n",
    "    end = time.time()\n",
    "\n",
    "    # printing out a status message - specifying the number of seconds elapsed\n",
    "    print(end-start, \"seconds elapsed\")\n",
    "\n",
    "    # adding all of the articles in the current chunk to storage\n",
    "    all_text.append(parsed_current_chunk)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(1915, 3)"
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 66,
   "metadata": {},
   "outputs": [
    {
     "ename": "AssertionError",
     "evalue": "",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mAssertionError\u001b[0m                            Traceback (most recent call last)",
      "Cell \u001b[0;32mIn[66], line 2\u001b[0m\n\u001b[1;32m      1\u001b[0m \u001b[38;5;66;03m# ensuring no extra columns have been created by any account\u001b[39;00m\n\u001b[0;32m----> 2\u001b[0m \u001b[38;5;28;01massert\u001b[39;00m \u001b[38;5;28msum\u001b[39m(\u001b[38;5;28mlen\u001b[39m(elem) \u001b[38;5;28;01mfor\u001b[39;00m elem \u001b[38;5;129;01min\u001b[39;00m all_text) \u001b[38;5;241m==\u001b[39m df\u001b[38;5;241m.\u001b[39mshape[\u001b[38;5;241m0\u001b[39m]\n",
      "\u001b[0;31mAssertionError\u001b[0m: "
     ]
    }
   ],
   "source": [
    "# ensuring no extra columns have been created by any account\n",
    "# assert sum(len(elem) for elem in all_text) == df.shape[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 69,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>title</th>\n",
       "      <th>link</th>\n",
       "      <th>text</th>\n",
       "      <th>article_type</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>Children and COVID-19 Vaccination Trends</td>\n",
       "      <td>https://www.aap.org/en/pages/2019-novel-corona...</td>\n",
       "      <td>Summary of data publicly reported by the Cente...</td>\n",
       "      <td>science</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>COVID-19 State-Level Data Reports</td>\n",
       "      <td>https://www.aap.org/en/pages/2019-novel-corona...</td>\n",
       "      <td>On May 11, 2023, the United States ended the P...</td>\n",
       "      <td>science</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>Prevention Papillomavirus can cause 6 types of...</td>\n",
       "      <td>https://www.cancer.org/cancer/risk-prevention/...</td>\n",
       "      <td>Our highly trained specialists are available 2...</td>\n",
       "      <td>science</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>COVID-19</td>\n",
       "      <td>https://www.lung.org/lung-health-diseases/lung...</td>\n",
       "      <td>Can we help you find more info? Start by selec...</td>\n",
       "      <td>science</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>End Youth Vaping Let\\'s join together to end t...</td>\n",
       "      <td>https://www.lung.org/quit-smoking/end-youth-va...</td>\n",
       "      <td>Research – Youth Vaping and Lung Health The Am...</td>\n",
       "      <td>science</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                               title  \\\n",
       "5           Children and COVID-19 Vaccination Trends   \n",
       "6                  COVID-19 State-Level Data Reports   \n",
       "7  Prevention Papillomavirus can cause 6 types of...   \n",
       "8                                           COVID-19   \n",
       "9  End Youth Vaping Let\\'s join together to end t...   \n",
       "\n",
       "                                                link  \\\n",
       "5  https://www.aap.org/en/pages/2019-novel-corona...   \n",
       "6  https://www.aap.org/en/pages/2019-novel-corona...   \n",
       "7  https://www.cancer.org/cancer/risk-prevention/...   \n",
       "8  https://www.lung.org/lung-health-diseases/lung...   \n",
       "9  https://www.lung.org/quit-smoking/end-youth-va...   \n",
       "\n",
       "                                                text article_type  \n",
       "5  Summary of data publicly reported by the Cente...      science  \n",
       "6  On May 11, 2023, the United States ended the P...      science  \n",
       "7  Our highly trained specialists are available 2...      science  \n",
       "8  Can we help you find more info? Start by selec...      science  \n",
       "9  Research – Youth Vaping and Lung Health The Am...      science  "
      ]
     },
     "execution_count": 69,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 70,
   "metadata": {},
   "outputs": [],
   "source": [
    "# creating a new column to store the texts for a particular article\n",
    "df['text'] = 'undetermined'\n",
    "\n",
    "# moving the article_type column to the far end of the dataframe, as where it should be\n",
    "df.insert(len(df.columns)-1, 'article_type', df.pop('article_type'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 71,
   "metadata": {},
   "outputs": [],
   "source": [
    "# creating a variable to store the current index in the all_text list\n",
    "current_index = 0\n",
    "\n",
    "# looping over all of the articles\n",
    "for current_chunk in all_text:\n",
    "\n",
    "    # looping over the articles parsed in the current chunk\n",
    "    for current_text in current_chunk:\n",
    "\n",
    "        # replacing the value in the current row with current_text\n",
    "        df.at[current_index, 'text'] = current_text\n",
    "\n",
    "        # incrementing the current_index\n",
    "        current_index += 1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 72,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>title</th>\n",
       "      <th>link</th>\n",
       "      <th>text</th>\n",
       "      <th>article_type</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>Children and COVID-19 Vaccination Trends</td>\n",
       "      <td>https://www.aap.org/en/pages/2019-novel-corona...</td>\n",
       "      <td>Summary of data publicly reported by the Cente...</td>\n",
       "      <td>science</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>COVID-19 State-Level Data Reports</td>\n",
       "      <td>https://www.aap.org/en/pages/2019-novel-corona...</td>\n",
       "      <td>On May 11, 2023, the United States ended the P...</td>\n",
       "      <td>science</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>Prevention Papillomavirus can cause 6 types of...</td>\n",
       "      <td>https://www.cancer.org/cancer/risk-prevention/...</td>\n",
       "      <td>Our highly trained specialists are available 2...</td>\n",
       "      <td>science</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>COVID-19</td>\n",
       "      <td>https://www.lung.org/lung-health-diseases/lung...</td>\n",
       "      <td>Can we help you find more info? Start by selec...</td>\n",
       "      <td>science</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>End Youth Vaping Let\\'s join together to end t...</td>\n",
       "      <td>https://www.lung.org/quit-smoking/end-youth-va...</td>\n",
       "      <td>Research – Youth Vaping and Lung Health\\n\\nThe...</td>\n",
       "      <td>science</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                               title  \\\n",
       "5           Children and COVID-19 Vaccination Trends   \n",
       "6                  COVID-19 State-Level Data Reports   \n",
       "7  Prevention Papillomavirus can cause 6 types of...   \n",
       "8                                           COVID-19   \n",
       "9  End Youth Vaping Let\\'s join together to end t...   \n",
       "\n",
       "                                                link  \\\n",
       "5  https://www.aap.org/en/pages/2019-novel-corona...   \n",
       "6  https://www.aap.org/en/pages/2019-novel-corona...   \n",
       "7  https://www.cancer.org/cancer/risk-prevention/...   \n",
       "8  https://www.lung.org/lung-health-diseases/lung...   \n",
       "9  https://www.lung.org/quit-smoking/end-youth-va...   \n",
       "\n",
       "                                                text article_type  \n",
       "5  Summary of data publicly reported by the Cente...      science  \n",
       "6  On May 11, 2023, the United States ended the P...      science  \n",
       "7  Our highly trained specialists are available 2...      science  \n",
       "8  Can we help you find more info? Start by selec...      science  \n",
       "9  Research – Youth Vaping and Lung Health\\n\\nThe...      science  "
      ]
     },
     "execution_count": 72,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 74,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "579"
      ]
     },
     "execution_count": 74,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# print out how many invalid articles there are\n",
    "df['text'].value_counts()['N/A']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 75,
   "metadata": {},
   "outputs": [],
   "source": [
    "# drop all columns with N/A in their text column\n",
    "df = df[df['text'] != \"N/A\"]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 76,
   "metadata": {},
   "outputs": [],
   "source": [
    "# drop all columns with no text\n",
    "df = df[df['text'] != \"\"]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 77,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>title</th>\n",
       "      <th>link</th>\n",
       "      <th>text</th>\n",
       "      <th>article_type</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>Children and COVID-19 Vaccination Trends</td>\n",
       "      <td>https://www.aap.org/en/pages/2019-novel-corona...</td>\n",
       "      <td>Summary of data publicly reported by the Cente...</td>\n",
       "      <td>science</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>COVID-19 State-Level Data Reports</td>\n",
       "      <td>https://www.aap.org/en/pages/2019-novel-corona...</td>\n",
       "      <td>On May 11, 2023, the United States ended the P...</td>\n",
       "      <td>science</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>Prevention Papillomavirus can cause 6 types of...</td>\n",
       "      <td>https://www.cancer.org/cancer/risk-prevention/...</td>\n",
       "      <td>Our highly trained specialists are available 2...</td>\n",
       "      <td>science</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>COVID-19</td>\n",
       "      <td>https://www.lung.org/lung-health-diseases/lung...</td>\n",
       "      <td>Can we help you find more info? Start by selec...</td>\n",
       "      <td>science</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>End Youth Vaping Let\\'s join together to end t...</td>\n",
       "      <td>https://www.lung.org/quit-smoking/end-youth-va...</td>\n",
       "      <td>Research – Youth Vaping and Lung Health\\n\\nThe...</td>\n",
       "      <td>science</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                               title  \\\n",
       "5           Children and COVID-19 Vaccination Trends   \n",
       "6                  COVID-19 State-Level Data Reports   \n",
       "7  Prevention Papillomavirus can cause 6 types of...   \n",
       "8                                           COVID-19   \n",
       "9  End Youth Vaping Let\\'s join together to end t...   \n",
       "\n",
       "                                                link  \\\n",
       "5  https://www.aap.org/en/pages/2019-novel-corona...   \n",
       "6  https://www.aap.org/en/pages/2019-novel-corona...   \n",
       "7  https://www.cancer.org/cancer/risk-prevention/...   \n",
       "8  https://www.lung.org/lung-health-diseases/lung...   \n",
       "9  https://www.lung.org/quit-smoking/end-youth-va...   \n",
       "\n",
       "                                                text article_type  \n",
       "5  Summary of data publicly reported by the Cente...      science  \n",
       "6  On May 11, 2023, the United States ended the P...      science  \n",
       "7  Our highly trained specialists are available 2...      science  \n",
       "8  Can we help you find more info? Start by selec...      science  \n",
       "9  Research – Youth Vaping and Lung Health\\n\\nThe...      science  "
      ]
     },
     "execution_count": 77,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "cleaning the data obtained to ensure <i>only</i> textual content is taken"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "- remove all escape sequences and non-ascii unicode characters (like \\\\xe2)\n",
    "- .strip() to remove any unnecessary spaces\n",
    "- standardize to single space between word"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 84,
   "metadata": {},
   "outputs": [],
   "source": [
    "import re\n",
    "\n",
    "# remove escape characters (\\n works, unicode characters stuff like \\\\xe90 doesn't work the best - e.g. the latter part of possessive or contractions are cut off)\n",
    "def clean_str(unfiltered_str: str):\n",
    "    formatted_str = re.sub('\\\\\\\\\\w+', '', unfiltered_str).strip()\n",
    "    return str((' ').join(formatted_str.split()))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 85,
   "metadata": {},
   "outputs": [],
   "source": [
    "# loop over all of the columns and and replace existing contents with cleaned up contents\n",
    "for i, row in df.iterrows():\n",
    "    df.at[i, 'title'] = clean_str(df.at[i, 'title'])\n",
    "    df.at[i, 'text'] = clean_str(df.at[i, 'text'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 86,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>title</th>\n",
       "      <th>link</th>\n",
       "      <th>text</th>\n",
       "      <th>article_type</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>Children and COVID-19 Vaccination Trends</td>\n",
       "      <td>https://www.aap.org/en/pages/2019-novel-corona...</td>\n",
       "      <td>Summary of data publicly reported by the Cente...</td>\n",
       "      <td>science</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>COVID-19 State-Level Data Reports</td>\n",
       "      <td>https://www.aap.org/en/pages/2019-novel-corona...</td>\n",
       "      <td>On May 11, 2023, the United States ended the P...</td>\n",
       "      <td>science</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>Prevention Papillomavirus can cause 6 types of...</td>\n",
       "      <td>https://www.cancer.org/cancer/risk-prevention/...</td>\n",
       "      <td>Our highly trained specialists are available 2...</td>\n",
       "      <td>science</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>COVID-19</td>\n",
       "      <td>https://www.lung.org/lung-health-diseases/lung...</td>\n",
       "      <td>Can we help you find more info? Start by selec...</td>\n",
       "      <td>science</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>End Youth Vaping Let\\'s join together to end t...</td>\n",
       "      <td>https://www.lung.org/quit-smoking/end-youth-va...</td>\n",
       "      <td>Research – Youth Vaping and Lung Health The Am...</td>\n",
       "      <td>science</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                               title  \\\n",
       "5           Children and COVID-19 Vaccination Trends   \n",
       "6                  COVID-19 State-Level Data Reports   \n",
       "7  Prevention Papillomavirus can cause 6 types of...   \n",
       "8                                           COVID-19   \n",
       "9  End Youth Vaping Let\\'s join together to end t...   \n",
       "\n",
       "                                                link  \\\n",
       "5  https://www.aap.org/en/pages/2019-novel-corona...   \n",
       "6  https://www.aap.org/en/pages/2019-novel-corona...   \n",
       "7  https://www.cancer.org/cancer/risk-prevention/...   \n",
       "8  https://www.lung.org/lung-health-diseases/lung...   \n",
       "9  https://www.lung.org/quit-smoking/end-youth-va...   \n",
       "\n",
       "                                                text article_type  \n",
       "5  Summary of data publicly reported by the Cente...      science  \n",
       "6  On May 11, 2023, the United States ended the P...      science  \n",
       "7  Our highly trained specialists are available 2...      science  \n",
       "8  Can we help you find more info? Start by selec...      science  \n",
       "9  Research – Youth Vaping and Lung Health The Am...      science  "
      ]
     },
     "execution_count": 86,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "saving the DataFrame for short-listed articles as JSON"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 87,
   "metadata": {},
   "outputs": [],
   "source": [
    "with open(\"data/articles.json\") as file:\n",
    "    current_data = json.load(file)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 88,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "1321"
      ]
     },
     "execution_count": 88,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(current_data)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 89,
   "metadata": {},
   "outputs": [],
   "source": [
    "extracted_articles = json.loads(df.to_json(orient='records'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 90,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>title</th>\n",
       "      <th>link</th>\n",
       "      <th>text</th>\n",
       "      <th>article_type</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>Children and COVID-19 Vaccination Trends</td>\n",
       "      <td>https://www.aap.org/en/pages/2019-novel-corona...</td>\n",
       "      <td>Summary of data publicly reported by the Cente...</td>\n",
       "      <td>science</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>COVID-19 State-Level Data Reports</td>\n",
       "      <td>https://www.aap.org/en/pages/2019-novel-corona...</td>\n",
       "      <td>On May 11, 2023, the United States ended the P...</td>\n",
       "      <td>science</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>Prevention Papillomavirus can cause 6 types of...</td>\n",
       "      <td>https://www.cancer.org/cancer/risk-prevention/...</td>\n",
       "      <td>Our highly trained specialists are available 2...</td>\n",
       "      <td>science</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>COVID-19</td>\n",
       "      <td>https://www.lung.org/lung-health-diseases/lung...</td>\n",
       "      <td>Can we help you find more info? Start by selec...</td>\n",
       "      <td>science</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>End Youth Vaping Let\\'s join together to end t...</td>\n",
       "      <td>https://www.lung.org/quit-smoking/end-youth-va...</td>\n",
       "      <td>Research – Youth Vaping and Lung Health The Am...</td>\n",
       "      <td>science</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                               title  \\\n",
       "5           Children and COVID-19 Vaccination Trends   \n",
       "6                  COVID-19 State-Level Data Reports   \n",
       "7  Prevention Papillomavirus can cause 6 types of...   \n",
       "8                                           COVID-19   \n",
       "9  End Youth Vaping Let\\'s join together to end t...   \n",
       "\n",
       "                                                link  \\\n",
       "5  https://www.aap.org/en/pages/2019-novel-corona...   \n",
       "6  https://www.aap.org/en/pages/2019-novel-corona...   \n",
       "7  https://www.cancer.org/cancer/risk-prevention/...   \n",
       "8  https://www.lung.org/lung-health-diseases/lung...   \n",
       "9  https://www.lung.org/quit-smoking/end-youth-va...   \n",
       "\n",
       "                                                text article_type  \n",
       "5  Summary of data publicly reported by the Cente...      science  \n",
       "6  On May 11, 2023, the United States ended the P...      science  \n",
       "7  Our highly trained specialists are available 2...      science  \n",
       "8  Can we help you find more info? Start by selec...      science  \n",
       "9  Research – Youth Vaping and Lung Health The Am...      science  "
      ]
     },
     "execution_count": 90,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 91,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[{'title': 'Children and COVID-19 Vaccination Trends',\n",
       "  'link': 'https://www.aap.org/en/pages/2019-novel-coronavirus-covid-19-infections/children-and-covid-19-vaccination-trends/',\n",
       "  'text': 'Summary of data publicly reported by the Centers for Disease Control and Prevention Date: 5/3/23 [Note: We will be updating this report every month, depending on the continued availability of the data.] The American Academy of Pediatrics recommends COVID-19 vaccination for all children and adolescents 6 months of age and older who do not have contraindications using a vaccine authorized for use for their age. See AAP policy. In this report, drawing on data posted by the Centers for Disease Control and Prevention (CDC) as of 5.3.23, the AAP reports progress in vaccinating US children, including the 50 States and District of Columbia. The report covers the vaccine data available for children 6 months-4 years, and the longer-term data for children 5-11 years and for children 12-17 years of age. See link below to download the full AAP report. Note: Individual state web sites may have additional or more recent information. Children Ages 6 Months-4 Years: As of May 3, 2023, the CDC recorded: 2.2 million US children ages 6 months-4 years have received at least one dose of COVID-19 vaccine Representing 13% of 6 months-4 year-olds This past week about 7,000 received their first vaccine dose. About 14.9 million children 6 months-4 year-olds had yet to receive their first COVID-19 vaccine dose. Child vaccination rates vary widely across states, ranging from 3% to 45% receiving their first dose. Children Ages 5-11 Years: 11.2 million US children ages 5-11 have received at least one dose of COVID-19 vaccine Representing 39% of 5-11 year-olds 9.2 million US children ages 5-11 completed the 2-dose vaccination series Representing 32% of 5-11 year-olds About 17.4 million children 5-11 had yet to receive their first COVID-19 vaccine dose. The past week about 6,000 received their first vaccine dose. Child vaccination rates vary widely across states, ranging from 17% to 83% receiving their first dose. Children Ages 12-17 Years: 17.9 million US children and adolescents ages 12- 17 have received at least one dose of COVID-19 vaccine Representing 68% of 12-17 year-olds 15.3 million of US children and adolescents ages 12- 17 completed the 2-dose vaccination series Representing 59% of 12-17 year-olds About 8.3 million children 12-17 had yet to receive their first COVID-19 vaccine dose. The past week about 5,000 received their first vaccine dose. Child vaccination rates vary widely across states, ranging from 40% to 100% receiving their first dose. Portion of US Children who have received vaccination: Across the total of US children 6 mos to 17 years, approximately 56%, 40.5 million children, have not received a first dose of the COVID-19 vaccine. Source: AAP analysis of data series titled “COVID -19 Vaccinations in the United States, Jurisdiction” as of May 3, 2023. Data cover the 50 States and District of Columbia. There are 73.6 M children under age 18 in the US, 22% of the population (Census 2021). The Pfizer-BioNTech COVID-19 vaccine was approved by the CDC in persons aged > 16 years on December 12, 2020, following an EUA from the FDA on December 11th. On May 12, 2021, CDC approved the use of the Pfizer-BioNTech COVID-19 vaccine in persons aged 12-15 years, following the vaccine’s EUA granted by the FDA on May 10th. The FDA issued an EUA for the Pfizer vaccine for kids aged 5 to 11 on 10.29.2021, followed by CDC recommendation on 11.2.2021. The FDA authorized the use of Moderna and Pfizer-BioNTech COVID-19 vaccines for children 6 months-4 years on 6.15.2022, followed by CDC recommendation on 6.18.2022. Download the Full Report 5/3/2023 Download the Full Report 4/5/2023 Download the Full Report 3/1/2023',\n",
       "  'article_type': 'science'},\n",
       " {'title': 'COVID-19 State-Level Data Reports',\n",
       "  'link': 'https://www.aap.org/en/pages/2019-novel-coronavirus-covid-19-infections/children-and-covid-19-state-level-data-report/',\n",
       "  'text': 'On May 11, 2023, the United States ended the Public Health Emergency (PHE) that was declared over three years ago to organize government resources and establish policies needed to respond to the COVID-19 pandemic. After three years of weekly reporting on the number of child COVID-19 cases in the United States, the American Academy of Pediatrics (AAP) and the Children’s Hospital Association (CHA) will be sunsetting the weekly state COVID-19 reports. Origins of the Report The report was created in the early months of the COVID-19 pandemic to measure how COVID-19 was affecting children in the United States. The CDC had begun providing national data on COVID-19 cases, but timely, regular data on pediatric cases were not available. Thus, we turned to data provided by states and territories and began to collect information from their web sites for the most current pediatric data. By compiling this information we could track the number of child cases weekly, as well as provide publicly reported case numbers for children at the state level. Changes in Data Available Over the past three years this report has provided a unique and valued resource to understand the impact of COVID-19 on children, and to understand trends in volume of cases and geography as COVID-19 variants emerged. The data in the report have always been limited by variations in how states categorized and reported COVID-19 cases. Beginning in 2022, as home COVID-19 tests became widely available, an increasing portion of COVID-19 cases were likely unreported. In 2023 an increasing number of states reduced the frequency of reporting and updating cases. As the end of the PHE approached, these trends intensified, and many states announced their posts were ending. We will continue to monitor COVID-19 with indicators that are available. However, given the decreasing access and reliability of data on the number of child COVID-19 infections, this weekly report was sunset May 11, 2023. We will continue disease surveillance as we do for other infectious diseases. Final Summary As of May 11, 2023, nearly 15.6 million children were reported to have tested positive for COVID-19 since the onset of the pandemic according to available state reports. Over the past three years, the highest number of child cases reported in a week was over 1.1 million for the week ending January 20, 2022. Since the pandemic began, children represented 17.9% of total cumulated cases. The portion of reported cases that were children ranged from a low of 2.0% the week ending April 16, 2020 to a high of 28.9% the week ending September 9, 2021 (children, under age 18, make up 22.2% of the US population). On May 11, an additional 6,848 child COVID-19 cases were reported. Over the past 6 months, weekly reported child cases have plateaued at an average of about 24,000 cases, while in the past month the average reported weekly cases have dropped further to about 9,000. These trends parallel the pattern and decline that has been reported for adults. It remains important to address the long-lasting impacts of the COVID-19 pandemic on the physical, mental, and social well-being of this generation of children and youth. The AAP compiled a snapshot of the impact of the COVID-19 pandemic on families and provides resources to help pediatricians support them. For a new online course on trauma-informed care and other recommendations, visit AAP.org/TIC.',\n",
       "  'article_type': 'science'},\n",
       " {'title': 'Prevention Papillomavirus can cause 6 types of cancer. But preventing it and the cancers it causes is simple.',\n",
       "  'link': 'https://www.cancer.org/cancer/risk-prevention/hpv/hpv-vaccine.html',\n",
       "  'text': 'Our highly trained specialists are available 24/7 via phone and on weekdays can assist through video calls and online chat. We connect patients, caregivers, and family members with essential services and resources at every step of their cancer journey. Ask us how you can get involved and support the fight against cancer. Some of the topics we can assist with include: Referrals to patient-related programs or resources Donations, website, or event-related assistance Tobacco-related topics Volunteer opportunities Cancer Information For medical questions, we encourage you to review our information with your doctor.',\n",
       "  'article_type': 'science'}]"
      ]
     },
     "execution_count": 91,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "extracted_articles[:3]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 92,
   "metadata": {},
   "outputs": [],
   "source": [
    "# adds new articles discovered to existing database (extending the database)\n",
    "def consolidate_data(existing_data, new_data):\n",
    "    def remove_dup(curr):\n",
    "        titles = set()\n",
    "        rev = []\n",
    "        for item in curr:\n",
    "            if item['title'] in titles:\n",
    "                continue\n",
    "            rev.append(item)\n",
    "            titles.add(item['title'])\n",
    "        return rev\n",
    "\n",
    "    with open(f\"data/articles.json\") as file:\n",
    "        existing_data = json.loads(file.read())\n",
    "\n",
    "    print(f\"Before combining: {len(existing_data)}\")\n",
    "    existing_data += new_data\n",
    "    existing_data = remove_dup(existing_data)\n",
    "    print(f\"After combining and removing duplicates: {len(existing_data)}\")\n",
    "\n",
    "    return remove_dup(existing_data)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 93,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Before combining: 1321\n",
      "After combining and removing duplicates: 1321\n"
     ]
    }
   ],
   "source": [
    "rev_data = consolidate_data(existing_data = current_data,\n",
    "                            new_data = extracted_articles)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 63,
   "metadata": {},
   "outputs": [],
   "source": [
    "with open(f\"data/articles.json\", 'w') as file:\n",
    "    json.dump(rev_data, file, indent = 4)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3.9.13 ('base')",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.13"
  },
  "orig_nbformat": 4,
  "vscode": {
   "interpreter": {
    "hash": "aa11b94746fc3e31dd444f69cbc07362dc8f0f9fc8892b4755141b10c2d19365"
   }
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}