make_user_repo_matrix.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import json\n",
    "import gzip\n",
    "import os\n",
    "from collections import Counter\n",
    "import datetime\n",
    "import sys\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "import scipy.sparse as sp\n",
    "import pdb"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "ename": "NameError",
     "evalue": "name 'gzip' is not defined",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mNameError\u001b[0m                                 Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-2-52102e91f3e6>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[1;32m     25\u001b[0m     \u001b[0;32mfor\u001b[0m \u001b[0mhour\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mrange\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mhour_start\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mhour_end\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     26\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 27\u001b[0;31m         \u001b[0;32mwith\u001b[0m \u001b[0mgzip\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mopen\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m'%s2018-07-%s-%d.json.gz'\u001b[0m \u001b[0;34m%\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mmy_path\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mmy_date\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mhour\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;32mas\u001b[0m \u001b[0mf\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     28\u001b[0m             \u001b[0;32mfor\u001b[0m \u001b[0mline\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mf\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     29\u001b[0m                 \u001b[0mevents\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mappend\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mjson\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mloads\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mline\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mstrip\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mNameError\u001b[0m: name 'gzip' is not defined"
     ]
    }
   ],
   "source": [
    "## read JSON data\n",
    "# change the file path accordingly\n",
    "# test data: file from wget http://data.gharchive.org/2018-07-16-15.json.gz\n",
    "# i.e. archive for 24 hours on 7/16/18\n",
    "\n",
    "######### CHANGE ME #############\n",
    "# specify date and hours for the archive data; assumes that the data are downloaded in the given path\n",
    "# also assumes the data is from july, 2018\n",
    "my_path = '/Users/amy/git/MetaGithub/data/' \n",
    "date_start = 16 # inclusive\n",
    "date_end = 17 # exclusive\n",
    "hour_start = 15 # inclusive\n",
    "hour_end = 16 # exclusive\n",
    "#################################\n",
    "\n",
    "events = []\n",
    "\n",
    "# for every date\n",
    "for date in range(date_start, date_end):\n",
    "    \n",
    "    # format date string so that 1 is encoded as '01', etc.\n",
    "    my_date = str(date) if date > 9 else '0'+ str(date)\n",
    "    \n",
    "    # for every hour\n",
    "    for hour in range(hour_start, hour_end):\n",
    "        \n",
    "        with gzip.open('%s2018-07-%s-%d.json.gz' %(my_path, my_date, hour)) as f:\n",
    "            for line in f:\n",
    "                events.append(json.loads(line.strip()))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "ename": "NameError",
     "evalue": "name 'events' is not defined",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mNameError\u001b[0m                                 Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-1-6e4f8b67e8fc>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[1;32m      1\u001b[0m \u001b[0;31m## get all watch events\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 2\u001b[0;31m \u001b[0mstar_events\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0mevent\u001b[0m \u001b[0;32mfor\u001b[0m \u001b[0mevent\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mevents\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0mevent\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m'type'\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;34m==\u001b[0m \u001b[0;34m'WatchEvent'\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m      3\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      4\u001b[0m \u001b[0;31m# aggregate starred repos for each user\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      5\u001b[0m \u001b[0muser2repo\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m{\u001b[0m\u001b[0;34m}\u001b[0m \u001b[0;31m# maps user name to list of repo names they've starred\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mNameError\u001b[0m: name 'events' is not defined"
     ]
    }
   ],
   "source": [
    "## get all watch events\n",
    "star_events = [event for event in events if event['type'] == 'WatchEvent']\n",
    "\n",
    "# aggregate starred repos for each user\n",
    "user2repo = {} # maps user name to list of repo names they've starred\n",
    "repos = set() # set of all repos that were starred\n",
    "user2id = {} # maps user name to its index\n",
    "repo2id = {} # maps repo name to its index\n",
    "user_id = 0 # user index starts at 0\n",
    "repo_id = 0 # repo index starts at 0\n",
    "\n",
    "for event in star_events:\n",
    "    \n",
    "    repo_name = event['repo']['name']\n",
    "    user_name = event['actor']['login']\n",
    "    \n",
    "    # for new user, update user2id and user_id\n",
    "    if user_name not in user2repo:\n",
    "        user2repo[user_name] = [repo_name]\n",
    "        user2id[user_name] = user_id\n",
    "        user_id += 1\n",
    "    else:\n",
    "        user2repo.get(user_name).append(repo_name)\n",
    "        \n",
    "    # for new repo, update repo2id and repo_id\n",
    "    if repo_name not in repos:\n",
    "        repos.add(repo_name)\n",
    "        repo2id[repo_name] = repo_id\n",
    "        repo_id += 1\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(4092, 4391)\n"
     ]
    }
   ],
   "source": [
    "## make a sparse matrix where entry (i,j) is 1 if the user i starred repo j\n",
    "n_user = len(user2repo)\n",
    "n_repo = len(repos)\n",
    "\n",
    "# initialie spare matrix\n",
    "mat = sp.dok_matrix((n_user,n_repo), dtype=np.int8)\n",
    "\n",
    "# set (i,j) = 1 if user i starred repo j\n",
    "for user_name, repo_names in user2repo.items():\n",
    "    \n",
    "    user_id = user2id[user_name]\n",
    "    repo_ids = [repo2id[r] for r in repo_names]\n",
    "\n",
    "    mat[user_id, repo_ids] = 1\n",
    "\n",
    "# convert to csr (compressed row sparse) matrix\n",
    "mat = mat.tocsr()\n",
    "print(mat.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style>\n",
       "    .dataframe thead tr:only-child th {\n",
       "        text-align: right;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: left;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>index</th>\n",
       "      <th>repo</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>didierfranc/react-waterfall</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1</td>\n",
       "      <td>Vscarlett/JQuery-stoplight</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2</td>\n",
       "      <td>Vscarlett/Personal-Express-Login</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>3</td>\n",
       "      <td>rickey-g/fancybear</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>4</td>\n",
       "      <td>Vscarlett/Name-Generator</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   index                              repo\n",
       "0      0       didierfranc/react-waterfall\n",
       "1      1        Vscarlett/JQuery-stoplight\n",
       "2      2  Vscarlett/Personal-Express-Login\n",
       "3      3                rickey-g/fancybear\n",
       "4      4          Vscarlett/Name-Generator"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# build dataframes we want to save: (i, users), (j, repos)\n",
    "\n",
    "# not efficient but whatever\n",
    "users = ['' for i in range(0, len(user2id))]\n",
    "for u, i in user2id.items():\n",
    "    users[i] = u\n",
    "\n",
    "repos = ['' for i in range(0, len(repo2id))]\n",
    "for r, i in repo2id.items():\n",
    "    repos[i] = r\n",
    "\n",
    "df_users = pd.DataFrame({'index': range(0, len(users)), 'user' : users})\n",
    "df_repos = pd.DataFrame({'index': range(0, len(repos)), 'repo' : repos})\n",
    "df_repos.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "# save files\n",
    "\n",
    "### CHANGE THIS PATH ACCORDINGLY ###\n",
    "out_path = '/Users/amy/git/MetaGithub/'\n",
    "\n",
    "# save users, repos, matrix\n",
    "df_users.to_csv(out_path + 'user_list.csv')\n",
    "df_repos.to_csv(out_path + 'repo_list.csv')\n",
    "sp.save_npz(out_path+\"user_repo_matrix.npz\", mat)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 70,
   "metadata": {},
   "outputs": [],
   "source": [
    "# loading the sparse matrix\n",
    "test = sp.load_npz(out_path+\"user_repo_matrix.npz\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}