notebooks/01_news_grabber.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import json\n",
    "import csv\n",
    "from newspaper import Article, fulltext\n",
    "\n",
    "import spacy\n",
    "from spacy import displacy\n",
    "\n",
    "from concurrent.futures import ThreadPoolExecutor, ProcessPoolExecutor, as_completed\n",
    "from tqdm import tqdm"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "# method to download articles\n",
    "def download_article(url):\n",
    "    try:\n",
    "        article = Article(url)\n",
    "        article.download()\n",
    "        return article\n",
    "    except Exception as e:\n",
    "        # print(f\"Exception occurred during downloading for URL {url}: {e}\")\n",
    "        return None\n",
    "\n",
    "# method to parse articles\n",
    "def parse_article(article):\n",
    "    try:\n",
    "        article.parse()\n",
    "        return [article.source_url, article.title, article.text.replace('\\n', ' '), \", \".join(article.authors), str(article.publish_date)]\n",
    "    except Exception as e:\n",
    "        # print(f\"Exception occurred during parsing: {e}\")\n",
    "        return [article.source_url, \"Parse Error\", None, None, None]\n",
    "\n",
    "\n",
    "# write to csv file\n",
    "def write_to_csv(filename, rows):\n",
    "    with open(filename, 'a', newline='') as f:\n",
    "        writer = csv.writer(f)\n",
    "        writer.writerows(rows)\n",
    "\n",
    "\n",
    "def process_batch(thread_executor, process_executor, urls, csv_file):\n",
    "    # Multithreaded downloading\n",
    "    future_to_url = {thread_executor.submit(download_article, url): url for url in urls}\n",
    "    download_results = [future.result() for future in as_completed(future_to_url) if future.result() is not None]\n",
    "\n",
    "    # Multiprocessing parsing\n",
    "    future_to_article = {process_executor.submit(parse_article, article): article for article in download_results}\n",
    "    parsed_results = [future.result() for future in as_completed(future_to_article)]\n",
    "\n",
    "    # Write to CSV\n",
    "    write_to_csv(csv_file, parsed_results)\n",
    "        \n",
    "# method to download and parse articles\n",
    "def get_article_text(channel):\n",
    "    json_file = channel + '_urls.json'\n",
    "    news_json = json.loads(open(json_file).read())\n",
    "    print(f\"Found {len(news_json)} articles for {channel}...\")\n",
    "    csv_file = channel + '_articles.csv'\n",
    "\n",
    "    # Initialize CSV file with headers\n",
    "    with open(csv_file, 'w', newline='') as f:\n",
    "        writer = csv.writer(f)\n",
    "        writer.writerow(['url', 'title', 'text', 'authors', 'publish_date'])\n",
    "\n",
    "    print(f\"Downloading and parsing articles for {channel}...\")\n",
    "    batch_size = 500\n",
    "    # Multithreaded downloading using 10 processors * 5 threads = 50 threads\n",
    "    # Multiprocessing parsing using 10 processors\n",
    "    with ThreadPoolExecutor(max_workers=50) as thread_executor, ProcessPoolExecutor() as process_executor:\n",
    "        for i in range(0, len(news_json), batch_size):\n",
    "            batch_urls = news_json[i:i+batch_size]\n",
    "            process_batch(thread_executor, process_executor, batch_urls, csv_file)\n",
    "            print(f\"Done with {i} articles\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Found 32958 articles for cnn...\n",
      "Downloading and parsing articles for cnn...\n",
      "Done with 0 articles\n",
      "Done with 500 articles\n",
      "Done with 1000 articles\n",
      "Done with 1500 articles\n",
      "Done with 2000 articles\n",
      "Done with 2500 articles\n",
      "Done with 3000 articles\n",
      "Done with 3500 articles\n",
      "Done with 4000 articles\n",
      "Done with 4500 articles\n",
      "Done with 5000 articles\n",
      "Done with 5500 articles\n",
      "Done with 6000 articles\n",
      "Done with 6500 articles\n",
      "Done with 7000 articles\n",
      "Done with 7500 articles\n",
      "Done with 8000 articles\n",
      "Done with 8500 articles\n",
      "Done with 9000 articles\n",
      "Done with 9500 articles\n",
      "Done with 10000 articles\n",
      "Done with 10500 articles\n",
      "Done with 11000 articles\n",
      "Done with 11500 articles\n",
      "Done with 12000 articles\n",
      "Done with 12500 articles\n",
      "Done with 13000 articles\n",
      "Done with 13500 articles\n",
      "Done with 14000 articles\n",
      "Done with 14500 articles\n",
      "Done with 15000 articles\n",
      "Done with 15500 articles\n",
      "Done with 16000 articles\n",
      "Done with 16500 articles\n",
      "Done with 17000 articles\n",
      "Done with 17500 articles\n",
      "Done with 18000 articles\n",
      "Done with 18500 articles\n",
      "Done with 19000 articles\n",
      "Done with 19500 articles\n",
      "Done with 20000 articles\n",
      "Done with 20500 articles\n",
      "Done with 21000 articles\n",
      "Done with 21500 articles\n",
      "Done with 22000 articles\n",
      "Done with 22500 articles\n",
      "Done with 23000 articles\n",
      "Done with 23500 articles\n",
      "Done with 24000 articles\n",
      "Done with 24500 articles\n",
      "Done with 25000 articles\n",
      "Done with 25500 articles\n",
      "Done with 26000 articles\n",
      "Done with 26500 articles\n",
      "Done with 27000 articles\n",
      "Done with 27500 articles\n",
      "Done with 28000 articles\n",
      "Done with 28500 articles\n",
      "Done with 29000 articles\n",
      "Done with 29500 articles\n",
      "Done with 30000 articles\n",
      "Done with 30500 articles\n",
      "Done with 31000 articles\n",
      "Done with 31500 articles\n",
      "Done with 32000 articles\n",
      "Done with 32500 articles\n",
      "CPU times: user 4h 18min 38s, sys: 1h 23min 54s, total: 5h 42min 32s\n",
      "Wall time: 3h 1min 46s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "\n",
    "articles = get_article_text('cnn')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Found 62738 articles for abc...\n",
      "Downloading and parsing articles for abc...\n",
      "Done with 0 articles\n",
      "Done with 500 articles\n",
      "Done with 1000 articles\n",
      "Done with 1500 articles\n",
      "Done with 2000 articles\n",
      "Done with 2500 articles\n",
      "Done with 3000 articles\n",
      "Done with 3500 articles\n",
      "Done with 4000 articles\n",
      "Done with 4500 articles\n",
      "Done with 5000 articles\n",
      "Done with 5500 articles\n",
      "Done with 6000 articles\n",
      "Done with 6500 articles\n",
      "Done with 7000 articles\n",
      "Done with 7500 articles\n",
      "Done with 8000 articles\n",
      "Done with 8500 articles\n",
      "Done with 9000 articles\n",
      "Done with 9500 articles\n",
      "Done with 10000 articles\n",
      "Done with 10500 articles\n",
      "Done with 11000 articles\n",
      "Done with 11500 articles\n",
      "Done with 12000 articles\n",
      "Done with 12500 articles\n",
      "Done with 13000 articles\n",
      "Done with 13500 articles\n",
      "Done with 14000 articles\n",
      "Done with 14500 articles\n",
      "Done with 15000 articles\n",
      "Done with 15500 articles\n",
      "Done with 16000 articles\n",
      "Done with 16500 articles\n",
      "Done with 17000 articles\n",
      "Done with 17500 articles\n",
      "Done with 18000 articles\n",
      "Done with 18500 articles\n",
      "Done with 19000 articles\n",
      "Done with 19500 articles\n",
      "Done with 20000 articles\n",
      "Done with 20500 articles\n",
      "Done with 21000 articles\n",
      "Done with 21500 articles\n",
      "Done with 22000 articles\n",
      "Done with 22500 articles\n",
      "Done with 23000 articles\n",
      "Done with 23500 articles\n",
      "Done with 24000 articles\n",
      "Done with 24500 articles\n",
      "Done with 25000 articles\n",
      "Done with 25500 articles\n",
      "Done with 26000 articles\n",
      "Done with 26500 articles\n",
      "Done with 27000 articles\n",
      "Done with 27500 articles\n",
      "Done with 28000 articles\n",
      "Done with 28500 articles\n",
      "Done with 29000 articles\n",
      "Done with 29500 articles\n",
      "Done with 30000 articles\n",
      "Done with 30500 articles\n",
      "Done with 31000 articles\n",
      "Done with 31500 articles\n",
      "Done with 32000 articles\n",
      "Done with 32500 articles\n",
      "Done with 33000 articles\n",
      "Done with 33500 articles\n",
      "Done with 34000 articles\n",
      "Done with 34500 articles\n",
      "Done with 35000 articles\n",
      "Done with 35500 articles\n",
      "Done with 36000 articles\n",
      "Done with 36500 articles\n",
      "Done with 37000 articles\n",
      "Done with 37500 articles\n",
      "Done with 38000 articles\n",
      "Done with 38500 articles\n",
      "Done with 39000 articles\n",
      "Done with 39500 articles\n",
      "Done with 40000 articles\n",
      "Done with 40500 articles\n",
      "Done with 41000 articles\n",
      "Done with 41500 articles\n",
      "Done with 42000 articles\n",
      "Done with 42500 articles\n",
      "Done with 43000 articles\n",
      "Done with 43500 articles\n",
      "Done with 44000 articles\n",
      "Done with 44500 articles\n",
      "Done with 45000 articles\n",
      "Done with 45500 articles\n",
      "Done with 46000 articles\n",
      "Done with 46500 articles\n",
      "Done with 47000 articles\n",
      "Done with 47500 articles\n",
      "Done with 48000 articles\n",
      "Done with 48500 articles\n",
      "Done with 49000 articles\n",
      "Done with 49500 articles\n",
      "Done with 50000 articles\n",
      "Done with 50500 articles\n",
      "Done with 51000 articles\n",
      "Done with 51500 articles\n",
      "Done with 52000 articles\n",
      "Done with 52500 articles\n",
      "Done with 53000 articles\n",
      "Done with 53500 articles\n",
      "Done with 54000 articles\n",
      "Done with 54500 articles\n",
      "Done with 55000 articles\n",
      "Done with 55500 articles\n",
      "Done with 56000 articles\n",
      "Done with 56500 articles\n",
      "Done with 57000 articles\n",
      "Done with 57500 articles\n",
      "Done with 58000 articles\n",
      "Done with 58500 articles\n",
      "Done with 59000 articles\n",
      "Done with 59500 articles\n",
      "Done with 60000 articles\n",
      "Done with 60500 articles\n",
      "Done with 61000 articles\n",
      "Done with 61500 articles\n",
      "Done with 62000 articles\n",
      "Done with 62500 articles\n",
      "CPU times: user 3h 20min 6s, sys: 49min 52s, total: 4h 9min 58s\n",
      "Wall time: 2h 15min 47s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "\n",
    "articles = get_article_text('abc')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Found 37907 articles for cbs...\n",
      "Downloading and parsing articles for cbs...\n",
      "Done with 0 articles\n",
      "Done with 500 articles\n",
      "Done with 1000 articles\n",
      "Done with 1500 articles\n",
      "Done with 2000 articles\n",
      "Done with 2500 articles\n",
      "Done with 3000 articles\n",
      "Done with 3500 articles\n",
      "Done with 4000 articles\n",
      "Done with 4500 articles\n",
      "Done with 5000 articles\n",
      "Done with 5500 articles\n",
      "Done with 6000 articles\n",
      "Done with 6500 articles\n",
      "Done with 7000 articles\n",
      "Done with 7500 articles\n",
      "Done with 8000 articles\n",
      "Done with 8500 articles\n",
      "Done with 9000 articles\n",
      "Done with 9500 articles\n",
      "Done with 10000 articles\n",
      "Done with 10500 articles\n",
      "Done with 11000 articles\n",
      "Done with 11500 articles\n",
      "Done with 12000 articles\n",
      "Done with 12500 articles\n",
      "Done with 13000 articles\n",
      "Done with 13500 articles\n",
      "Done with 14000 articles\n",
      "Done with 14500 articles\n",
      "Done with 15000 articles\n",
      "Done with 15500 articles\n",
      "Done with 16000 articles\n",
      "Done with 16500 articles\n",
      "Done with 17000 articles\n",
      "Done with 17500 articles\n",
      "Done with 18000 articles\n",
      "Done with 18500 articles\n",
      "Done with 19000 articles\n",
      "Done with 19500 articles\n",
      "Done with 20000 articles\n",
      "Done with 20500 articles\n",
      "Done with 21000 articles\n",
      "Done with 21500 articles\n",
      "Done with 22000 articles\n",
      "Done with 22500 articles\n",
      "Done with 23000 articles\n",
      "Done with 23500 articles\n",
      "Done with 24000 articles\n",
      "Done with 24500 articles\n",
      "Done with 25000 articles\n",
      "Done with 25500 articles\n",
      "Done with 26000 articles\n",
      "Done with 26500 articles\n",
      "Done with 27000 articles\n",
      "Done with 27500 articles\n",
      "Done with 28000 articles\n",
      "Done with 28500 articles\n",
      "Done with 29000 articles\n",
      "Done with 29500 articles\n",
      "Done with 30000 articles\n",
      "Done with 30500 articles\n",
      "Done with 31000 articles\n",
      "Done with 31500 articles\n",
      "Done with 32000 articles\n",
      "Done with 32500 articles\n",
      "Done with 33000 articles\n",
      "Done with 33500 articles\n",
      "Done with 34000 articles\n",
      "Done with 34500 articles\n",
      "Done with 35000 articles\n",
      "Done with 35500 articles\n",
      "Done with 36000 articles\n",
      "Done with 36500 articles\n",
      "Done with 37000 articles\n",
      "Done with 37500 articles\n",
      "CPU times: user 2h 20min 8s, sys: 37min 6s, total: 2h 57min 15s\n",
      "Wall time: 1h 41min 13s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "\n",
    "articles = get_article_text('cbs')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Found 17968 articles for nbc...\n",
      "Downloading and parsing articles for nbc...\n",
      "Done with 0 articles\n",
      "Done with 500 articles\n",
      "Done with 1000 articles\n",
      "Done with 1500 articles\n",
      "Done with 2000 articles\n",
      "Done with 2500 articles\n",
      "Done with 3000 articles\n",
      "Done with 3500 articles\n",
      "Done with 4000 articles\n",
      "Done with 4500 articles\n",
      "Done with 5000 articles\n",
      "Done with 5500 articles\n",
      "Done with 6000 articles\n",
      "Done with 6500 articles\n",
      "Done with 7000 articles\n",
      "Done with 7500 articles\n",
      "Done with 8000 articles\n",
      "Done with 8500 articles\n",
      "Done with 9000 articles\n",
      "Done with 9500 articles\n",
      "Done with 10000 articles\n",
      "Done with 10500 articles\n",
      "Done with 11000 articles\n",
      "Done with 11500 articles\n",
      "Done with 12000 articles\n",
      "Done with 12500 articles\n",
      "Done with 13000 articles\n",
      "Done with 13500 articles\n",
      "Done with 14000 articles\n",
      "Done with 14500 articles\n",
      "Done with 15000 articles\n",
      "Done with 15500 articles\n",
      "Done with 16000 articles\n",
      "Done with 16500 articles\n",
      "Done with 17000 articles\n",
      "Done with 17500 articles\n",
      "CPU times: user 1h 9min 18s, sys: 19min 53s, total: 1h 29min 11s\n",
      "Wall time: 51min\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "\n",
    "articles = get_article_text('nbc')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Found 15108 articles for npr...\n",
      "Downloading and parsing articles for npr...\n",
      "Done with 0 articles\n",
      "Done with 500 articles\n",
      "Done with 1000 articles\n",
      "Done with 1500 articles\n",
      "Done with 2000 articles\n",
      "Done with 2500 articles\n",
      "Done with 3000 articles\n",
      "Done with 3500 articles\n",
      "Done with 4000 articles\n",
      "Done with 4500 articles\n",
      "Done with 5000 articles\n",
      "Done with 5500 articles\n",
      "Done with 6000 articles\n",
      "Done with 6500 articles\n",
      "Done with 7000 articles\n",
      "Done with 7500 articles\n",
      "Done with 8000 articles\n",
      "Done with 8500 articles\n",
      "Done with 9000 articles\n",
      "Done with 9500 articles\n",
      "Done with 10000 articles\n",
      "Done with 10500 articles\n",
      "Done with 11000 articles\n",
      "Done with 11500 articles\n",
      "Done with 12000 articles\n",
      "Done with 12500 articles\n",
      "Done with 13000 articles\n",
      "Done with 13500 articles\n",
      "Done with 14000 articles\n",
      "Done with 14500 articles\n",
      "Done with 15000 articles\n",
      "CPU times: user 59min 22s, sys: 16min 47s, total: 1h 16min 10s\n",
      "Wall time: 41min\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "\n",
    "articles = get_article_text('npr')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Found 26738 articles for nyt...\n",
      "Downloading and parsing articles for nyt...\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Building prefix dict from /home/distill1994/workspace/top_news/.venv/lib/python3.10/site-packages/jieba/dict.txt ...\n",
      "Building prefix dict from /home/distill1994/workspace/top_news/.venv/lib/python3.10/site-packages/jieba/dict.txt ...\n",
      "Building prefix dict from /home/distill1994/workspace/top_news/.venv/lib/python3.10/site-packages/jieba/dict.txt ...\n",
      "Dumping model to file cache /tmp/jieba.cache\n",
      "Dumping model to file cache /tmp/jieba.cache\n",
      "Dumping model to file cache /tmp/jieba.cache\n",
      "Loading model cost 9.437125444412231 seconds.\n",
      "Prefix dict has been built succesfully.\n",
      "Loading model cost 8.908550262451172 seconds.\n",
      "Prefix dict has been built succesfully.\n",
      "Loading model cost 7.9084861278533936 seconds.\n",
      "Prefix dict has been built succesfully.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Done with 0 articles\n",
      "Done with 500 articles\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Building prefix dict from /home/distill1994/workspace/top_news/.venv/lib/python3.10/site-packages/jieba/dict.txt ...\n",
      "Loading model from cache /tmp/jieba.cache\n",
      "Loading model cost 18.981618881225586 seconds.\n",
      "Prefix dict has been built succesfully.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Done with 1000 articles\n",
      "Done with 1500 articles\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Building prefix dict from /home/distill1994/workspace/top_news/.venv/lib/python3.10/site-packages/jieba/dict.txt ...\n",
      "Loading model from cache /tmp/jieba.cache\n",
      "Loading model cost 1.8977100849151611 seconds.\n",
      "Prefix dict has been built succesfully.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Done with 2000 articles\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Building prefix dict from /home/distill1994/workspace/top_news/.venv/lib/python3.10/site-packages/jieba/dict.txt ...\n",
      "Loading model from cache /tmp/jieba.cache\n",
      "Loading model cost 15.222313404083252 seconds.\n",
      "Prefix dict has been built succesfully.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Done with 2500 articles\n",
      "Done with 3000 articles\n",
      "Done with 3500 articles\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Building prefix dict from /home/distill1994/workspace/top_news/.venv/lib/python3.10/site-packages/jieba/dict.txt ...\n",
      "Loading model from cache /tmp/jieba.cache\n",
      "Loading model cost 18.67027711868286 seconds.\n",
      "Prefix dict has been built succesfully.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Done with 4000 articles\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Building prefix dict from /home/distill1994/workspace/top_news/.venv/lib/python3.10/site-packages/jieba/dict.txt ...\n",
      "Loading model from cache /tmp/jieba.cache\n",
      "Loading model cost 4.090705394744873 seconds.\n",
      "Prefix dict has been built succesfully.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Done with 4500 articles\n",
      "Done with 5000 articles\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Building prefix dict from /home/distill1994/workspace/top_news/.venv/lib/python3.10/site-packages/jieba/dict.txt ...\n",
      "Loading model from cache /tmp/jieba.cache\n",
      "Loading model cost 27.434269666671753 seconds.\n",
      "Prefix dict has been built succesfully.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Done with 5500 articles\n",
      "Done with 6000 articles\n",
      "Done with 6500 articles\n",
      "Done with 7000 articles\n",
      "Done with 7500 articles\n",
      "Done with 8000 articles\n",
      "Done with 8500 articles\n",
      "Done with 9000 articles\n",
      "Done with 9500 articles\n",
      "Done with 10000 articles\n",
      "Done with 10500 articles\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Building prefix dict from /home/distill1994/workspace/top_news/.venv/lib/python3.10/site-packages/jieba/dict.txt ...\n",
      "Loading model from cache /tmp/jieba.cache\n",
      "Loading model cost 12.549663305282593 seconds.\n",
      "Prefix dict has been built succesfully.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Done with 11000 articles\n",
      "Done with 11500 articles\n",
      "Done with 12000 articles\n",
      "Done with 12500 articles\n",
      "Done with 13000 articles\n",
      "Done with 13500 articles\n",
      "Done with 14000 articles\n",
      "Done with 14500 articles\n",
      "Done with 15000 articles\n",
      "Done with 15500 articles\n",
      "Done with 16000 articles\n",
      "Done with 16500 articles\n",
      "Done with 17000 articles\n",
      "Done with 17500 articles\n",
      "Done with 18000 articles\n",
      "Done with 18500 articles\n",
      "Done with 19000 articles\n",
      "Done with 19500 articles\n",
      "Done with 20000 articles\n",
      "Done with 20500 articles\n",
      "Done with 21000 articles\n",
      "Done with 21500 articles\n",
      "Done with 22000 articles\n",
      "Done with 22500 articles\n",
      "Done with 23000 articles\n",
      "Done with 23500 articles\n",
      "Done with 24000 articles\n",
      "Done with 24500 articles\n",
      "Done with 25000 articles\n",
      "Done with 25500 articles\n",
      "Done with 26000 articles\n",
      "Done with 26500 articles\n",
      "CPU times: user 1h 53min 50s, sys: 33min 53s, total: 2h 27min 44s\n",
      "Wall time: 1h 20min 27s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "\n",
    "articles = get_article_text('nyt')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Found 40637 articles for usat...\n",
      "Downloading and parsing articles for usat...\n",
      "Done with 0 articles\n",
      "Done with 500 articles\n",
      "Done with 1000 articles\n",
      "Done with 1500 articles\n",
      "Done with 2000 articles\n",
      "Done with 2500 articles\n",
      "Done with 3000 articles\n",
      "Done with 3500 articles\n",
      "Done with 4000 articles\n",
      "Done with 4500 articles\n",
      "Done with 5000 articles\n",
      "Done with 5500 articles\n",
      "Done with 6000 articles\n",
      "Done with 6500 articles\n",
      "Done with 7000 articles\n",
      "Done with 7500 articles\n",
      "Done with 8000 articles\n",
      "Done with 8500 articles\n",
      "Done with 9000 articles\n",
      "Done with 9500 articles\n",
      "Done with 10000 articles\n",
      "Done with 10500 articles\n",
      "Done with 11000 articles\n",
      "Done with 11500 articles\n",
      "Done with 12000 articles\n",
      "Done with 12500 articles\n",
      "Done with 13000 articles\n",
      "Done with 13500 articles\n",
      "Done with 14000 articles\n",
      "Done with 14500 articles\n",
      "Done with 15000 articles\n",
      "Done with 15500 articles\n",
      "Done with 16000 articles\n",
      "Done with 16500 articles\n",
      "Done with 17000 articles\n",
      "Done with 17500 articles\n",
      "Done with 18000 articles\n",
      "Done with 18500 articles\n",
      "Done with 19000 articles\n",
      "Done with 19500 articles\n",
      "Done with 20000 articles\n",
      "Done with 20500 articles\n",
      "Done with 21000 articles\n",
      "Done with 21500 articles\n",
      "Done with 22000 articles\n",
      "Done with 22500 articles\n",
      "Done with 23000 articles\n",
      "Done with 23500 articles\n",
      "Done with 24000 articles\n",
      "Done with 24500 articles\n",
      "Done with 25000 articles\n",
      "Done with 25500 articles\n",
      "Done with 26000 articles\n",
      "Done with 26500 articles\n",
      "Done with 27000 articles\n",
      "Done with 27500 articles\n",
      "Done with 28000 articles\n",
      "Done with 28500 articles\n",
      "Done with 29000 articles\n",
      "Done with 29500 articles\n",
      "Done with 30000 articles\n",
      "Done with 30500 articles\n",
      "Done with 31000 articles\n",
      "Done with 31500 articles\n",
      "Done with 32000 articles\n",
      "Done with 32500 articles\n",
      "Done with 33000 articles\n",
      "Done with 33500 articles\n",
      "Done with 34000 articles\n",
      "Done with 34500 articles\n",
      "Done with 35000 articles\n",
      "Done with 35500 articles\n",
      "Done with 36000 articles\n",
      "Done with 36500 articles\n",
      "Done with 37000 articles\n",
      "Done with 37500 articles\n",
      "Done with 38000 articles\n",
      "Done with 38500 articles\n",
      "Done with 39000 articles\n",
      "Done with 39500 articles\n",
      "Done with 40000 articles\n",
      "Done with 40500 articles\n",
      "CPU times: user 5h 25min 54s, sys: 1h 30min 12s, total: 6h 56min 7s\n",
      "Wall time: 3h 32min 12s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "\n",
    "articles = get_article_text('usat')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Found 33764 articles for wapo...\n",
      "Downloading and parsing articles for wapo...\n",
      "Done with 0 articles\n",
      "Done with 500 articles\n",
      "Done with 1000 articles\n",
      "Done with 1500 articles\n",
      "Done with 2000 articles\n",
      "Done with 2500 articles\n",
      "Done with 3000 articles\n",
      "Done with 3500 articles\n",
      "Done with 4000 articles\n",
      "Done with 4500 articles\n",
      "Done with 5000 articles\n",
      "Done with 5500 articles\n",
      "Done with 6000 articles\n",
      "Done with 6500 articles\n",
      "Done with 7000 articles\n",
      "Done with 7500 articles\n",
      "Done with 8000 articles\n",
      "Done with 8500 articles\n",
      "Done with 9000 articles\n",
      "Done with 9500 articles\n",
      "Done with 10000 articles\n",
      "Done with 10500 articles\n",
      "Done with 11000 articles\n",
      "Done with 11500 articles\n",
      "Done with 12000 articles\n",
      "Done with 12500 articles\n",
      "Done with 13000 articles\n",
      "Done with 13500 articles\n",
      "Done with 14000 articles\n",
      "Done with 14500 articles\n",
      "Done with 15000 articles\n",
      "Done with 15500 articles\n",
      "Done with 16000 articles\n",
      "Done with 16500 articles\n",
      "Done with 17000 articles\n",
      "Done with 17500 articles\n",
      "Done with 18000 articles\n",
      "Done with 18500 articles\n",
      "Done with 19000 articles\n",
      "Done with 19500 articles\n",
      "Done with 20000 articles\n",
      "Done with 20500 articles\n",
      "Done with 21000 articles\n",
      "Done with 21500 articles\n",
      "Done with 22000 articles\n",
      "Done with 22500 articles\n",
      "Done with 23000 articles\n",
      "Done with 23500 articles\n",
      "Done with 24000 articles\n",
      "Done with 24500 articles\n",
      "Done with 25000 articles\n",
      "Done with 25500 articles\n",
      "Done with 26000 articles\n",
      "Done with 26500 articles\n",
      "Done with 27000 articles\n",
      "Done with 27500 articles\n",
      "Done with 28000 articles\n",
      "Done with 28500 articles\n",
      "Done with 29000 articles\n",
      "Done with 29500 articles\n",
      "Done with 30000 articles\n",
      "Done with 30500 articles\n",
      "Done with 31000 articles\n",
      "Done with 31500 articles\n",
      "Done with 32000 articles\n",
      "Done with 32500 articles\n",
      "Done with 33000 articles\n",
      "Done with 33500 articles\n",
      "CPU times: user 2h 42min 8s, sys: 54min 37s, total: 3h 36min 46s\n",
      "Wall time: 1h 59min 50s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "\n",
    "articles = get_article_text('wapo')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Found 2456 articles for politico...\n",
      "Downloading and parsing articles for politico...\n",
      "Done with 0 articles\n",
      "Done with 500 articles\n",
      "Done with 1000 articles\n",
      "Done with 1500 articles\n",
      "Done with 2000 articles\n",
      "CPU times: user 9min 40s, sys: 4min 1s, total: 13min 42s\n",
      "Wall time: 7min 13s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "\n",
    "articles = get_article_text('politico')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Found 559 articles for propub...\n",
      "Downloading and parsing articles for propub...\n",
      "Done with 0 articles\n",
      "Done with 500 articles\n",
      "CPU times: user 2min 27s, sys: 1min 1s, total: 3min 28s\n",
      "Wall time: 1min 51s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "\n",
    "articles = get_article_text('propub')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Found 19510 articles for lat...\n",
      "Downloading and parsing articles for lat...\n",
      "Done with 0 articles\n",
      "Done with 500 articles\n",
      "Done with 1000 articles\n",
      "Done with 1500 articles\n",
      "Done with 2000 articles\n",
      "Done with 2500 articles\n",
      "Done with 3000 articles\n",
      "Done with 3500 articles\n",
      "Done with 4000 articles\n",
      "Done with 4500 articles\n",
      "Done with 5000 articles\n",
      "Done with 5500 articles\n",
      "Done with 6000 articles\n",
      "Done with 6500 articles\n",
      "Done with 7000 articles\n",
      "Done with 7500 articles\n",
      "Done with 8000 articles\n",
      "Done with 8500 articles\n",
      "Done with 9000 articles\n",
      "Done with 9500 articles\n",
      "Done with 10000 articles\n",
      "Done with 10500 articles\n",
      "Done with 11000 articles\n",
      "Done with 11500 articles\n",
      "Done with 12000 articles\n",
      "Done with 12500 articles\n",
      "Done with 13000 articles\n",
      "Done with 13500 articles\n",
      "Done with 14000 articles\n",
      "Done with 14500 articles\n",
      "Done with 15000 articles\n",
      "Done with 15500 articles\n",
      "Done with 16000 articles\n",
      "Done with 16500 articles\n",
      "Done with 17000 articles\n",
      "Done with 17500 articles\n",
      "Done with 18000 articles\n",
      "Done with 18500 articles\n",
      "Done with 19000 articles\n",
      "Done with 19500 articles\n",
      "CPU times: user 1h 20min 28s, sys: 24min 13s, total: 1h 44min 42s\n",
      "Wall time: 57min 58s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "\n",
    "articles = get_article_text('lat')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": ".venv",
   "language": "python",
   "name": ".venv"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}