articleScraperML.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "outputs": [],
   "source": [
    "import requests\n",
    "from bs4 import BeautifulSoup\n",
    "import pandas as pd\n",
    "import time\n",
    "import sys\n",
    "from datetime import date, datetime, timedelta\n"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n",
     "is_executing": false
    }
   }
  },
  {
   "cell_type": "markdown",
   "source": [
    "##### Entering the search query, from and to date - to get the news articles in that period"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%% md\n"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "outputs": [
    {
     "name": "stdout",
     "text": [
      "2019-12-29\n"
     ],
     "output_type": "stream"
    }
   ],
   "source": [
    "def getQuery(str):\n",
    "    result = str.split()\n",
    "    return result\n",
    "query = getQuery('champions league')\n",
    "# print(query)\n",
    "start = date(2019, 12, 29)\n",
    "end = date(2019, 12, 30)\n",
    "print(start)\n"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n",
     "is_executing": false
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "outputs": [],
   "source": [
    "# creating a function for getting a url\n",
    "def createUrl(query, month, day, year):\n",
    "    tempQ = []\n",
    "    len_a = len(query)\n",
    "    if (len_a > 1):\n",
    "        for i in range(len_a):\n",
    "            tempQ.append(query[i])        \n",
    "        \n",
    "        sep = '+'\n",
    "        tempQ1 = sep.join(tempQ)\n",
    "        url = 'https://www.google.com/search?q=' + str(tempQ1) + '&sxsrf=ACYBGNTwLQ0U4gEoAyzjmWThzJK06MJhxw%3A1577815826486&source=lnt&tbs=cdr%3A1%2Ccd_min%3A' + str(month) + '%2F' + str(day) + '%2F' + str(year) + '%2Ccd_max%3A' + str(month) + '%2F' + str(day) + '%2F' + str(year) + '&tbm=nws'\n",
    "    \n",
    "    else:\n",
    "        url = 'https://www.google.com/search?q=' + str(query) + '&sxsrf=ACYBGNTwLQ0U4gEoAyzjmWThzJK06MJhxw%3A1577815826486&source=lnt&tbs=cdr%3A1%2Ccd_min%3A' + str(month) + '%2F' + str(day) + '%2F' + str(year) + '%2Ccd_max%3A' + str(month) + '%2F' + str(day) + '%2F' + str(year) + '&tbm=nws'\n",
    "        \n",
    "    return url\n",
    "\n",
    "# curl = createUrl(query, month=12, day=31, year=2019)\n",
    "# curl"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n",
     "is_executing": false
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "outputs": [
    {
     "name": "stdout",
     "text": [
      "<generator object dateRange at 0x7f4fffb696d0>\n"
     ],
     "output_type": "stream"
    }
   ],
   "source": [
    "# defininig a function for getting a range of dates\n",
    "def dateRange(start, end):\n",
    "    temp = (end - start).days\n",
    "    # print(temp)\n",
    "    for i in range(int(temp)):\n",
    "        yield (start + timedelta(i))\n",
    "\n",
    "print(dateRange(start, end))"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n",
     "is_executing": false
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "outputs": [],
   "source": [
    "# function for getting the headlines using urllib and Beautiful Soup\n",
    "def captureHeadlines(page):\n",
    "    \n",
    "    # A user agent is string that each web browser sends to the website. It contains info.\n",
    "    # like the device software, OS, etc. so that the remote web servers use this data\n",
    "    # to optimize the webpage's performance and display.\n",
    "    userAgent = 'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/72.0.3579.0 Safari/537.36'\n",
    "    \n",
    "    headers = {\"User-Agent\": userAgent}\n",
    "    response = requests.get(page, headers = headers)\n",
    "    # print(response.content)\n",
    "    \n",
    "    soup = BeautifulSoup(response.content, 'html.parser')\n",
    "    headlinesCollec = []\n",
    "    # hdls = soup.find_all('div', attrs={'class': 'bkWMgd'})\n",
    "    # hdls1 = soup.find_all('div', attrs={'class': 'dbsr'})\n",
    "    hdls1 = soup.find_all('div', attrs={'class': 'g'})\n",
    "    for data in hdls1:\n",
    "        print('\\n')\n",
    "        temp = data.find('a')\n",
    "        # dataObj = {\n",
    "        #     \"Title\": data.find('div', attrs={'class': 'phYMDf nDgy9d'}).get_text(),\n",
    "        #     \"Publisher\": data.find('div', attrs={'class': 'pDavDe RGRr8e'}).get_text(),\n",
    "        #     \"Link\": temp['href']\n",
    "        # }\n",
    "        dataObj = {\n",
    "            \"Title\": data.find('h3', attrs={'class': 'r dO0Ag'}).get_text(),\n",
    "            \"Publisher\": data.find('span', attrs={'class': 'xQ82C'}).get_text(),\n",
    "            \"Link\": temp['href']\n",
    "        }\n",
    "\n",
    "        headlinesCollec.append(dataObj)            \n",
    "    return headlinesCollec  \n",
    "\n",
    "# captureHeadlines('https://www.google.com/search?q=arsenal&biw=1366&bih=669&sxsrf=ACYBGNTLcO_bWTajN0YjR_4rxoHl9GVo7g%3A1577820127297&source=lnt&tbs=cdr%3A1%2Ccd_min%3A12%2F29%2F2019%2Ccd_max%3A12%2F30%2F2019&tbm=nws')\n"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n",
     "is_executing": false
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "outputs": [],
   "source": [
    "def getHeadlines(query, start, end):\n",
    "    \n",
    "    urlList = []\n",
    "    for dates in dateRange(start,end):\n",
    "        # print(dates)\n",
    "        currentDate = dates.strftime(\"%m-%d-%Y\").split('-')\n",
    "        # print(currentDate)\n",
    "        url = createUrl(query, currentDate[0], currentDate[1], currentDate[2])\n",
    "        # print(url)\n",
    "        urlList.append(url)\n",
    "        \n",
    "    headlines = []\n",
    "    for u in urlList:\n",
    "        headL = captureHeadlines(u)\n",
    "        headlines.append(headL)\n",
    "        time.sleep(3.0)\n",
    "        \n",
    "    return headlines"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n",
     "is_executing": false
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "outputs": [
    {
     "name": "stdout",
     "text": [
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n"
     ],
     "output_type": "stream"
    },
    {
     "data": {
      "text/plain": "[[{'Title': \"Erling Braut Haaland: Dortmund's new signing by numbers\",\n   'Publisher': 'UEFA.com',\n   'Link': 'https://www.uefa.com/uefachampionsleague/news/newsid=2637609.html'},\n  {'Title': 'Fernandinho: Manchester City need to fight for Champions ...',\n   'Publisher': 'Sky Sports',\n   'Link': 'https://www.skysports.com/football/news/11679/11897125/fernandinho-manchester-city-need-to-fight-for-champions-league-spot'},\n  {'Title': \"Liverpool to wear gold 'FIFA World Champions' badge during ...\",\n   'Publisher': 'GiveMeSport',\n   'Link': 'https://www.givemesport.com/1533530-liverpool-to-wear-gold-fifa-world-champions-badge-during-their-match-against-wolves'},\n  {'Title': 'Top 4 in sight: Can Manchester United get into the Champions ...',\n   'Publisher': 'The Standard',\n   'Link': 'https://www.standardmedia.co.ke/sports/article/2001354691/top-4-in-sight-can-manchester-united-get-into-the-champions-league'},\n  {'Title': 'Chelsea comes back to win 2-1 at Arsenal in Premier League',\n   'Publisher': 'USA TODAY',\n   'Link': 'https://www.usatoday.com/story/sports/soccer/2019/12/29/chelsea-comes-back-to-win-2-1-at-arsenal-in-premier-league/40902463/'},\n  {'Title': 'Manchester City focus turns to cups and top-four finish',\n   'Publisher': 'RTE.ie',\n   'Link': 'https://www.rte.ie/sport/soccer/2019/1229/1103526-manchester-city-focus-turns-to-cups-and-top-four-finish/'},\n  {'Title': 'African Champions League: Al Hilal stun Etoile and el Kaabi ...',\n   'Publisher': 'BBC News',\n   'Link': 'https://www.bbc.co.uk/sport/football/50935778'},\n  {'Title': 'Sundowns still have to fight to reach Champions League ...',\n   'Publisher': 'HeraldLIVE',\n   'Link': 'https://www.heraldlive.co.za/sport/2019-12-29-sundowns-still-have-to-fight-to-reach-champions-league-quarters-warns-mosimane/'},\n  {'Title': \"Guardiola: It's 'amazing' to have set a new Premier League ...\",\n   'Publisher': 'NBC Sports - Misc. (blog)',\n   'Link': 'https://soccer.nbcsports.com/2019/12/29/pep-guardiola-manchester-city-premier-league-record-wenger-ferguson-klopp-mourinho/'},\n  {'Title': 'Managers who went back as David Moyes rejoins West Ham',\n   'Publisher': 'FourFourTwo USA (blog)',\n   'Link': 'https://www.fourfourtwo.com/us/news/managers-who-went-back-david-moyes-rejoins-west-ham'}]]"
     },
     "metadata": {},
     "output_type": "execute_result",
     "execution_count": 7
    }
   ],
   "source": [
    "getHeadlines(query, start, end)"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n",
     "is_executing": false
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "outputs": [],
   "source": [
    "\n"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.6"
  },
  "pycharm": {
   "stem_cell": {
    "cell_type": "raw",
    "source": [],
    "metadata": {
     "collapsed": false
    }
   }
  }
 },
 "nbformat": 4,
 "nbformat_minor": 0
}