data_exploration_amy.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "import json\n",
    "import gzip\n",
    "import os\n",
    "from collections import Counter\n",
    "from github import Github\n",
    "import datetime\n",
    "import sys\n",
    "import pickle\n",
    "from matplotlib import pyplot as plt\n",
    "import pandas as pd"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "## load arvhive data\n",
    "events = []\n",
    "# File from: wget http://data.gharchive.org/2018-07-06-{0..23}.json.gz\n",
    "# i.e. archive for 24 hours on 7/16/18\n",
    "for hour in range(0, 24):\n",
    "    with gzip.open('/Users/amy/git/MetaGithub/data/2018-07-16-%d.json.gz' %hour) as f:\n",
    "        for line in f:\n",
    "            events.append(json.loads(line.strip()))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Counter({'CommitCommentEvent': 2262,\n",
       "         'CreateEvent': 183861,\n",
       "         'DeleteEvent': 30330,\n",
       "         'ForkEvent': 38550,\n",
       "         'GollumEvent': 6843,\n",
       "         'IssueCommentEvent': 93487,\n",
       "         'IssuesEvent': 48759,\n",
       "         'MemberEvent': 6089,\n",
       "         'PublicEvent': 1402,\n",
       "         'PullRequestEvent': 81249,\n",
       "         'PullRequestReviewCommentEvent': 29417,\n",
       "         'PushEvent': 698124,\n",
       "         'ReleaseEvent': 5771,\n",
       "         'WatchEvent': 107964})"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "Counter((event['type'] for event in events))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'actor': {'avatar_url': 'https://avatars.githubusercontent.com/u/8587468?',\n",
       "  'display_login': 'dhruvdhody-huawei',\n",
       "  'gravatar_id': '',\n",
       "  'id': 8587468,\n",
       "  'login': 'dhruvdhody-huawei',\n",
       "  'url': 'https://api.github.com/users/dhruvdhody-huawei'},\n",
       " 'created_at': '2018-07-16T00:00:00Z',\n",
       " 'id': '7968171066',\n",
       " 'payload': {'before': '28d1b1ea1ac223abc32f10615481774665fbc374',\n",
       "  'commits': [{'author': {'email': 'c1299ee609067fe82b3838a8d655fb28525d1b02@gmail.com',\n",
       "     'name': 'Dhruv Dhody'},\n",
       "    'distinct': True,\n",
       "    'message': 'Add files via upload',\n",
       "    'sha': '966109e8910165dc2e16e4831596cff2249ebc7b',\n",
       "    'url': 'https://api.github.com/repos/dhruvdhody-huawei/102/commits/966109e8910165dc2e16e4831596cff2249ebc7b'}],\n",
       "  'distinct_size': 1,\n",
       "  'head': '966109e8910165dc2e16e4831596cff2249ebc7b',\n",
       "  'push_id': 2721067372,\n",
       "  'ref': 'refs/heads/master',\n",
       "  'size': 1},\n",
       " 'public': True,\n",
       " 'repo': {'id': 139835450,\n",
       "  'name': 'dhruvdhody-huawei/102',\n",
       "  'url': 'https://api.github.com/repos/dhruvdhody-huawei/102'},\n",
       " 'type': 'PushEvent'}"
      ]
     },
     "execution_count": 21,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "events[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [],
   "source": [
    "# get all watch events\n",
    "# The WatchEvent is related to starring a repository, not watching. \n",
    "starred = [event for event in events if event['type'] == 'WatchEvent']\n",
    "\n",
    "# count how many times each repo was starred during this time\n",
    "# maps repo name --> count of star event\n",
    "repo2count = {}\n",
    "\n",
    "for event in starred:\n",
    "    \n",
    "    repo_name = event['repo']['name']\n",
    "    \n",
    "    if repo_name not in repo2count:\n",
    "        repo2count[repo_name] = 1\n",
    "    else:\n",
    "        repo2count[repo_name] += 1\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# get all commit comment events\n",
    "comment_events = [event for event in events if event['type'] == 'CommitCommentEvent']\n",
    "\n",
    "# aggregate comments for each repo in this timeframe\n",
    "# maps repo_name --> list of comments made during this time \n",
    "repo2comments = {}\n",
    "\n",
    "for event in comment_events:\n",
    "    \n",
    "    repo_name = event['repo']['name']\n",
    "    comment = event['payload']['comment']['body']\n",
    "    \n",
    "    if repo_name not in repo2comments:\n",
    "        repo2comments[repo_name] = [comment]\n",
    "    else:\n",
    "        repo2comments.get(repo_name).append(comment)\n",
    "    \n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# save repo2comment dictionary\n",
    "my_file = open('/Users/amy/git/MetaGithub/data/repo2commitComments.7.16.2018.pickle', 'wb')\n",
    "pickle.dump(repo2comments, my_file)\n",
    "my_file.close()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "1032"
      ]
     },
     "execution_count": 23,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# record the names of repos that were starred more than k times during this time\n",
    "popular_repos = []\n",
    "k = 10\n",
    "\n",
    "for repo_name in repo2count:\n",
    "    if repo2count[repo_name] > k:\n",
    "        popular_repos.append(repo_name)\n",
    "        \n",
    "len(popular_repos)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 222,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# for each popular repo, count the number of (+1, -1, laugh, confused, heart, hooray, no_reaction, num_issues) on issues\n",
    "# Exceeds API rate limit on github\n",
    "\n",
    "# github object using my personal access token\n",
    "g = Github('dc346ff7262b4b55dcc8ea6abe35c9c3170d415b')\n",
    "\n",
    "# map repo name --> counts of [+1, -1, laugh, confused, heart, hooray, no_reaction, num_issues]\n",
    "repo2info = {}\n",
    "my_date = datetime.datetime(year = 2018, month = 7, day = 1) # starting time from which to collect data \n",
    "\n",
    "for repo_name in popular_repos:\n",
    "    \n",
    "    # get the repository\n",
    "    repo = g.get_repo(repo_name)\n",
    "    \n",
    "    # get the issues for this repository\n",
    "    issue_comments = repo.get_issues_comments(since = my_date)\n",
    "    \n",
    "    # initialize count dictionary for this repo\n",
    "    repo2info[repo_name] = {'+1':0, '-1':0, 'laugh':0, 'confused':0, 'heart':0, 'hooray':0, 'no_reaction': 0, 'num_comments_with_no_reaction':0, 'num_comments':0}\n",
    "\n",
    "    # for each issue, get reactions\n",
    "    for comment in issue_comments:\n",
    "        \n",
    "        info = repo2info[repo_name]\n",
    "        \n",
    "        # get reactions for this repo\n",
    "        reactions = comment.get_reactions()\n",
    "        \n",
    "        # count each reaction\n",
    "        for r in reactions:\n",
    "            info[r.content] += 1\n",
    "            \n",
    "        # count number of comments without any reactions\n",
    "        if len(reactions._PaginatedListBase__elements) == 0:\n",
    "            info['num_comments_with_no_reaction'] += 1\n",
    "            \n",
    "        # increment number of comments\n",
    "        info['num_comments'] += 1\n",
    "\n",
    "            "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "# compare avg sentiment vs. n_stars\n",
    "\n",
    "my_file = open('/Users/amy/git/MetaGithub/data/repo2sentiment.7.16.2018.pickle', 'rb')\n",
    "repo2sentiment = pickle.load(my_file)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "# make data frame\n",
    "names = []\n",
    "n_stars = []\n",
    "avg_sent = []\n",
    "n_comments = []\n",
    "for repo_name in repo2sentiment.keys():\n",
    "    \n",
    "    names.append(repo_name)\n",
    "    if repo_name in repo2count:\n",
    "        n_stars.append(repo2count[repo_name])\n",
    "    else:\n",
    "        n_stars.append(0)\n",
    "        \n",
    "    \n",
    "    count = repo2sentiment[repo_name]\n",
    "    total_sum = 0\n",
    "    n = 0\n",
    "    for key in count:\n",
    "        total_sum += (count[key] * int(key))\n",
    "        n += count[key]\n",
    "    if n != 0:\n",
    "        avg_sent.append(total_sum / n)\n",
    "    else:\n",
    "        avg_sent.append(None)\n",
    "    n_comments.append(n)\n",
    "    \n",
    "df = pd.DataFrame({'name': names, 'n_stars':n_stars, 'n_commit_comments': n_comments, 'avg_commit_comment_sentiment':avg_sent})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [],
   "source": [
    "df.head()\n",
    "df.to_csv('/Users/amy/git/MetaGithub/data/df.7.16.17.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}