diff --git a/backend/Procfile b/backend/Procfile deleted file mode 100755 index 09c5658..0000000 --- a/backend/Procfile +++ /dev/null @@ -1 +0,0 @@ -web: python ghstats.py diff --git a/backend/config.json.sample b/backend/config.json.sample deleted file mode 100644 index 55167d7..0000000 --- a/backend/config.json.sample +++ /dev/null @@ -1,32 +0,0 @@ -{ -"AppID": { - "clientId": "your App ID client Id", - "managementUrl": "https://appid-management.ng.bluemix.net/management/v4/-----tenantID----", - "oauthServerUrl": "https://appid-oauth.ng.bluemix.net/oauth/v3/-----tenantID----", - "profilesUrl": "https://appid-profiles.ng.bluemix.net", - "secret": "the App ID secret", - "tenantId": "-----tenantID----", - "version": 3 -}, -"dashDB": -{ - "db": "BLUDB", - "dsn": "DATABASE=BLUDB;HOSTNAME=dashdb------.services.---.bluemix.net;PORT=50000;PROTOCOL=TCPIP;UID=dashxxxxx;PWD=xxxxxxxx;", - "host": "dashdb-------.services.---.bluemix.net", - "hostname": "dashdb------.services.---.bluemix.net", - "https_url": "https://dashdb-----------.bluemix.net:8443", - "jdbcurl": "jdbc:db2://dashdb---------------.bluemix.net:50000/BLUDB", - "password": "the password", - "port": 50000, - "ssldsn": "DATABASE=BLUDB;HOSTNAME=dashdb-------------------.bluemix.net;PORT=50001;PROTOCOL=TCPIP;UID=dashxxxxx;PWD=xxxxxxxxx;Security=SSL;", - "ssljdbcurl": "jdbc:db2://dashdb------------------.bluemix.net:50001/BLUDB:sslConnection=true;", - "uri": "db2://dashxxxxx:xxxxxxxxxxx@dashdb---------------.bluemix.net:50000/BLUDB", - "username": "dashxxxxx" - }, -"dynamic-dashboard-embedded": { - "api_endpoint_url": "https://dde-us-south.analytics.ibm.com/daas/", - "apikey": "xxxxxxxxxxxxxxx", - "client_id": "yyyyyyyyyyyyyy", - "client_secret": "zzzzzzzzzzzzzzzzz", -} -} diff --git a/backend/dashboard.json b/backend/dashboard.json deleted file mode 100644 index 2e9b285..0000000 --- a/backend/dashboard.json +++ /dev/null @@ -1,284 +0,0 @@ -{ - "name": "GitHub Traffic", - "layout": { - "id": "page0", - "items": [ - { - "id": "page1", - "css": "templateBox aspectRatio_default", - "items": [ - { - "id": "page2", - "style": { - "top": "0%", - "left": "0%", - "right": "0%", - "bottom": "75%" - }, - "type": "templateDropZone", - "templateName": "dz1", - "relatedLayouts": "|model00000162d427dcfa_00000000|" - }, - { - "id": "page3", - "css": "noBorderTop", - "style": { - "top": "25%", - "left": "0%", - "right": "0%", - "bottom": "0%" - }, - "type": "templateDropZone", - "templateName": "dz2", - "relatedLayouts": "|model00000162d42896ca_00000000|" - }, - { - "id": "model00000162d42896ca_00000000", - "style": { - "top": "25.09157509157509%", - "left": "0.1006036217303823%", - "height": "74.9084249084249%", - "width": "99.79879275653923%" - }, - "type": "widget", - "relatedLayouts": "page3" - }, - { - "id": "model00000162d427dcfa_00000000", - "style": { - "top": "0.18315018315018314%", - "left": "0.1006036217303823%", - "height": "24.725274725274726%", - "width": "99.79879275653923%" - }, - "type": "widget", - "relatedLayouts": "page2" - } - ], - "type": "scalingAbsolute" - } - ], - "type": "container", - "templateName": "Template2" - }, - "theme": "defaultTheme", - "version": 1008, - "eventGroups": [ - { - "id": "page1:1", - "widgetIds": [ - "model00000162d427dcfa_00000000", - "model00000162d42896ca_00000000" - ] - } - ], - "dataSources": { - "version": "1.0", - "sources": [ - { - "id": "model00000162d427a660_00000002", - "assetId": "assetId00000162d427a660_00000000", - "clientId": "Repostats", - "module": { - "identifier": "Repostats", - "label": "Repository Traffic Data", - "source": { - "id": "Repostats", - "srcUrl": { - "mimeType": "text/csv", - "property": [ - { - "name": "separator", - "value": ", " - }, - { - "name": "ColumnNamesLine", - "value": "true" - }, - { - "name": "headers", - "value": [ - { - "name": "Authorization", - "value": "Note: dynamically configured" - } - ] - } - ], - "sourceUrl": "Note: dynamically configured" - } - }, - "table": { - "column": [ - { - "datatype": "INTEGER", - "description": "repository ID", - "label": "Repository ID", - "name": "RID", - "nullable": "false", - "regularAggregate": "countDistinct", - "usage": "identifier" - }, - { - "datatype": "VARCHAR(255)", - "description": "Organization or user", - "label": "organization or user", - "name": "ORGNAME", - "nullable": "false", - "regularAggregate": "countDistinct", - "usage": "identifier" - }, - { - "datatype": "VARCHAR(255)", - "description": "repository name", - "label": "repository name", - "name": "REPONAME", - "nullable": "false", - "regularAggregate": "total", - "usage": "fact" - }, - { - "datatype": "DATE", - "description": "traffic date", - "label": "traffic date", - "name": "TDATE", - "nullable": "false", - "regularAggregate": "countDistinct", - "taxonomyFamily": "cDate", - "usage": "identifier" - }, - { - "datatype": "INTEGER", - "label": "count of views", - "name": "VIEWCOUNT", - "nullable": "false", - "regularAggregate": "total", - "usage": "fact" - }, - { - "datatype": "INTEGER", - "label": "unique views", - "name": "VUNIQUES", - "nullable": "false", - "regularAggregate": "total", - "usage": "fact" - }, - { - "datatype": "INTEGER", - "label": "count of clones", - "name": "CLONECOUNT", - "nullable": "false", - "regularAggregate": "total", - "usage": "fact" - }, - { - "datatype": "INTEGER", - "label": "unique counts", - "name": "CUNIQUES", - "nullable": "false", - "regularAggregate": "total", - "usage": "fact" - } - ], - "description": "Traffic data for repositories", - "name": "repositorystats" - }, - "xsd": "https://ibm.com/daas/module/1.0/module.xsd" - }, - "name": "Traffic CSV", - "shaping": { - "embeddedModuleUpToDate": true - } - } - ] - }, - "pageContext": [], - "drillThrough": [], - "widgets": { - "model00000162d427dcfa_00000000": { - "id": "model00000162d427dcfa_00000000", - "data": { - "dataViews": [ - { - "modelRef": "model00000162d427a660_00000002", - "dataItems": [ - { - "id": "model00000162d42806aa_00000000", - "itemId": "repositorystats.RID", - "itemLabel": "Repository ID" - } - ], - "id": "model00000162d42806a8_00000000" - } - ] - }, - "visTypeLocked": true, - "slotmapping": { - "slots": [ - { - "name": "values", - "dataItems": [ - "model00000162d42806aa_00000000" - ], - "dataItemSettings": [], - "caption": "Value", - "id": "values" - } - ] - }, - "type": "live", - "visId": "summary", - "name": "" - }, - "model00000162d42896ca_00000000": { - "id": "model00000162d42896ca_00000000", - "data": { - "dataViews": [ - { - "modelRef": "model00000162d427a660_00000002", - "dataItems": [ - { - "id": "model00000162d42962ee_00000000", - "itemId": "repositorystats.VIEWCOUNT", - "itemLabel": "count of views" - }, - { - "id": "model00000162d42b7ba4_00000000", - "itemId": "repositorystats.TDATE", - "itemLabel": "traffic date" - } - ], - "id": "model00000162d428c4ea_00000000" - } - ] - }, - "visTypeLocked": true, - "slotmapping": { - "slots": [ - { - "name": "values", - "dataItems": [ - "model00000162d42962ee_00000000" - ], - "caption": "y-axis", - "id": "values", - "layerId": "data" - }, - { - "name": "categories", - "dataItems": [ - "model00000162d42b7ba4_00000000" - ], - "dataItemSettings": [], - "caption": "x-axis", - "id": "categories" - } - ] - }, - "type": "live", - "visId": "com.ibm.vis.rave2line", - "name": "", - "localFilters": [] - } - } -} diff --git a/backend/dashboard2.json b/backend/dashboard2.json deleted file mode 100644 index 6acc3ef..0000000 --- a/backend/dashboard2.json +++ /dev/null @@ -1 +0,0 @@ -{"name":"New dashboard","layout":{"id":"page0","items":[{"id":"page1","items":[{"id":"page2","css":"templateBox aspectRatio_default","items":[{"id":"page3","style":{"top":"0%","left":"0%","right":"0%","bottom":"0%"},"type":"templateDropZone","relatedLayouts":"|model00000162ddc1b400_00000000|"},{"id":"model00000162ddc1b400_00000000","style":{"top":"0.1865671641791045%","left":"0.10256410256410256%","height":"99.6268656716418%","width":"99.7948717948718%"},"type":"widget","relatedLayouts":"page3"}],"type":"scalingAbsolute"}],"type":"container","title":"Tab 1","templateName":"Template1"},{"id":"model00000162ddc2826c_00000000","items":[{"id":"model00000162ddc2826e_00000000","css":"templateBox aspectRatio_default","items":[{"id":"model00000162ddc2826e_00000001","style":{"top":"0%","left":"0%","right":"0%","bottom":"0%"},"type":"templateDropZone","relatedLayouts":"|model00000162ddc29e8e_00000000|"},{"id":"model00000162ddc29e8e_00000000","style":{"top":"0.1865671641791045%","left":"0.10256410256410256%","height":"99.6268656716418%","width":"99.7948717948718%"},"type":"widget","relatedLayouts":"model00000162ddc2826e_00000001"}],"type":"scalingAbsolute"}],"type":"container","title":"Tab 2","templateName":"Template1"}],"style":{"height":"100%"},"type":"tab"},"theme":"defaultTheme","version":1008,"eventGroups":[{"id":"page1:1","widgetIds":["model00000162ddc1b400_00000000"]},{"id":"model00000162ddc2826c_00000000:1","widgetIds":["model00000162ddc29e8e_00000000"]}],"dataSources":{"version":"1.0","sources":[{"id":"model00000162ddc185b0_00000002","assetId":"assetId00000162ddc185b0_00000000","clientId":"Repostats","module":{"identifier":"Repostats","label":"Repository Traffic Data","source":{"id":"Repostats","srcUrl":{"mimeType":"text/csv","property":[{"name":"separator","value":", "},{"name":"ColumnNamesLine","value":"true"},{"name":"headers","value":[{"name":"Authorization","value":"Basic ZXlKaGJHY2lPaUpJVXpJMU5pSXNJbVY0Y0NJNk1UVXlOREUwTWpRek5pd2lhV0YwSWpveE5USTBNVE00T0RNMmZRLmV5SnBaQ0k2SW1oc2IyVnpaWEpBWkdVdWFXSnRMbU52YlNKOS5IcTc0ZGJFY0FLQ053OFVId2NLZkQ2bnNQaVljWEZWaW5wZmE4V1VPVXJrOklhbWF0b2tlbg=="}]}],"sourceUrl":"http://github-traffic-stats.mybluemix.net/api/v1/data/repositorystats.csv"}},"table":{"column":[{"datatype":"INTEGER","description":"repository ID","label":"Repository ID","name":"RID","nullable":"false","regularAggregate":"countDistinct","usage":"identifier"},{"datatype":"VARCHAR(255)","description":"Organization or user","label":"organization or user","name":"ORGNAME","nullable":"false","regularAggregate":"countDistinct","usage":"identifier"},{"datatype":"VARCHAR(255)","description":"repository name","label":"repository name","name":"REPONAME","nullable":"false","regularAggregate":"total","usage":"fact"},{"datatype":"DATE","description":"traffic date","label":"traffic date","name":"TDATE","nullable":"false","regularAggregate":"countDistinct","taxonomyFamily":"cDate","usage":"identifier"},{"datatype":"INTEGER","label":"count of views","name":"VIEWCOUNT","nullable":"false","regularAggregate":"total","usage":"fact"},{"datatype":"INTEGER","label":"unique views","name":"VUNIQUES","nullable":"false","regularAggregate":"total","usage":"fact"},{"datatype":"INTEGER","label":"count of clones","name":"CLONECOUNT","nullable":"false","regularAggregate":"total","usage":"fact"},{"datatype":"INTEGER","label":"unique counts","name":"CUNIQUES","nullable":"false","regularAggregate":"total","usage":"fact"}],"description":"Traffic data for repositories","name":"repositorystats"},"xsd":"https://ibm.com/daas/module/1.0/module.xsd"},"name":"Traffic CSV","shaping":{"embeddedModuleUpToDate":true}}]},"pageContext":[],"drillThrough":[],"widgets":{"model00000162ddc1b400_00000000":{"id":"model00000162ddc1b400_00000000","data":{"dataViews":[{"modelRef":"model00000162ddc185b0_00000002","dataItems":[{"id":"model00000162ddc1d466_00000000","itemId":"repositorystats.TDATE","itemLabel":"traffic date"},{"id":"model00000162ddc1e258_00000000","itemId":"repositorystats.VIEWCOUNT","itemLabel":"count of views"},{"id":"model00000162ddc1ff5a_00000000","itemId":"repositorystats.CLONECOUNT","itemLabel":"count of clones"},{"id":"_multiMeasuresSeries","itemId":"_multiMeasuresSeries","itemLabel":"Measures group (2)"}],"id":"model00000162ddc1d464_00000000"}]},"visTypeLocked":true,"slotmapping":{"slots":[{"name":"categories","dataItems":["model00000162ddc1d466_00000000"],"dataItemSettings":[],"caption":"Bars","id":"categories"},{"name":"values","dataItems":["model00000162ddc1ff5a_00000000","model00000162ddc1e258_00000000"],"caption":"Length","id":"values"},{"name":"color","dataItems":["_multiMeasuresSeries"],"caption":"Color","id":"color"}]},"type":"live","visId":"com.ibm.vis.rave2bundlestackedcolumn","name":""},"model00000162ddc29e8e_00000000":{"id":"model00000162ddc29e8e_00000000","data":{"dataViews":[{"modelRef":"model00000162ddc185b0_00000002","dataItems":[{"id":"model00000162ddc2b924_00000000","itemId":"repositorystats.TDATE","itemLabel":"traffic date"},{"id":"model00000162ddc2e09c_00000000","itemId":"repositorystats.VIEWCOUNT","itemLabel":"count of views"},{"id":"model00000162ddc568a2_00000000","itemId":"repositorystats.REPONAME","itemLabel":"repository name"}],"id":"model00000162ddc2b922_00000000"}]},"visTypeLocked":true,"slotmapping":{"slots":[{"name":"categories","dataItems":["model00000162ddc2b924_00000000"],"dataItemSettings":[],"caption":"x-axis","id":"categories"},{"name":"values","dataItems":["model00000162ddc2e09c_00000000"],"caption":"y-axis","id":"values"},{"name":"series","dataItems":["model00000162ddc568a2_00000000"],"caption":"Color","id":"series"}]},"type":"live","visId":"com.ibm.vis.rave2line","name":"","localFilters":[]}}} diff --git a/backend/manifest.yml b/backend/manifest.yml deleted file mode 100644 index 5b7fd28..0000000 --- a/backend/manifest.yml +++ /dev/null @@ -1,9 +0,0 @@ -applications: -- name: github-traffic-stats - memory: 128M - command: python ghstats.py - random-route: true - services: - - ghstatsDB - - ghstatsAppID - - ghstatsDDE diff --git a/backend/runtime.txt b/backend/runtime.txt deleted file mode 100644 index 795ee72..0000000 --- a/backend/runtime.txt +++ /dev/null @@ -1 +0,0 @@ -python-3.7.9 diff --git a/functions/__main__.py b/functions/__main__.py deleted file mode 100644 index 1bff801..0000000 --- a/functions/__main__.py +++ /dev/null @@ -1,130 +0,0 @@ -####### -####### GitHub traffic statistics -####### -# Program or IBM Cloud Function to collect GitHub view and clone traffic -# and store that data in a relational database, namely Db2. -# -# (C) 2018 IBM -# Written by Henrik Loeser, hloeser@de.ibm.com ("data-henrik") - -import github, json, ibm_db, sys, time - -####### -# SQL statements -# -# fetch all users -allTenantsStatement="select tid, ghuser, ghtoken from tenants" -# fetch all repos for a given userID -allReposStatement="select r.rid, ghu.username, r.rname from tenantrepos tr,repos r, ghorgusers ghu where tr.rid=r.rid and r.oid=ghu.oid and tr.tid=?" - -# merge the view traffic data -mergeViews1="merge into repotraffic rt using (values" -mergeViews2=""") as nv(rid,viewdate,viewcount,uniques) on rt.rid=nv.rid and rt.tdate=nv.viewdate - when matched and nv.viewcount>rt.viewcount then update set viewcount=nv.viewcount, vuniques=coalesce(nv.uniques,0) - when not matched then insert (rid,tdate,viewcount, vuniques) values(nv.rid,nv.viewdate,coalesce(nv.viewcount,0),coalesce(nv.uniques,0)) - else ignore""" - -# merge the clone traffic data -mergeClones1="merge into repotraffic rt using (values" -mergeClones2=""") as nc(rid,clonedate,clonecount,uniques) on rt.rid=nc.rid and rt.tdate=nc.clonedate - when matched and nc.clonecount>rt.clonecount then update set clonecount=nc.clonecount, cuniques=coalesce(nc.uniques,0) - when not matched then insert (rid,tdate,clonecount,cuniques) values(nc.rid,nc.clonedate,coalesce(nc.clonecount,0),coalesce(nc.uniques,0)) - else ignore""" - -# new syslog record -insertLogEntry="insert into systemlog values(?,?,?,?)" - -# Merge view data into the traffic table -def mergeViewData(viewStats, rid): - # convert traffic data into SQL values - data="" - for vday in viewStats['views']: - data+="("+str(rid)+",'"+vday['timestamp'][:10]+"',"+str(vday['count'])+","+str(vday['uniques'])+")," - mergeStatement=mergeViews1+data[:-1]+mergeViews2 - # execute MERGE statement - res=ibm_db.exec_immediate(conn,mergeStatement) - -# Merge clone data into the traffic table -def mergeCloneData(cloneStats, rid): - # convert traffic data into SQL values - data="" - for cday in cloneStats['clones']: - data+="("+str(rid)+",'"+cday['timestamp'][:10]+"',"+str(cday['count'])+","+str(cday['uniques'])+")," - mergeStatement=mergeClones1+data[:-1]+mergeClones2 - # execute MERGE statement - res=ibm_db.exec_immediate(conn,mergeStatement) - - -# Overall flow: -# - loop over users -# - log in to GitHub as that current user -# - retrieve repos for that current user, loop the repos -# - for each repo fetch stats -# - merge traffic data into table -# update last run info - -def main(args): - global conn - repoCount=0 - processedRepos=0 - logtext="cloudfunction (" - errortext="" - - ssldsn = args["__bx_creds"]["dashDB"]["ssldsn"] - #ssldsn = args["ssldsn"] - if globals().get("conn") is None: - conn = ibm_db.connect(ssldsn, "", "") - - # go over all system users - allTenants=ibm_db.exec_immediate(conn,allTenantsStatement) - if (allTenants): - - # prepare statement for logging - logStmt = ibm_db.prepare(conn, insertLogEntry) - - # fetch first user - tenant=ibm_db.fetch_assoc(allTenants) - while tenant != False: - # go over all repos managed by that user and fetch traffic data - # first, login to GitHub as that user - gh = github.GitHub(username=tenant["GHUSER"], access_token=tenant["GHTOKEN"]) - - userRepoCount=0 - # prepare and execute statement to fetch related repositories - reposStmt = ibm_db.prepare(conn, allReposStatement) - if (ibm_db.execute(reposStmt,(tenant["TID"],))): - repo=ibm_db.fetch_assoc(reposStmt) - while repo != False: - repoCount=repoCount+1 - # fetch view and clone traffic - try: - viewStats=gh.repos(repo["USERNAME"], repo["RNAME"]).traffic.views.get() - cloneStats=gh.repos(repo["USERNAME"], repo["RNAME"]).traffic.clones.get() - if viewStats['views']: - mergeViewData(viewStats,repo["RID"]) - if cloneStats['clones']: - mergeCloneData(cloneStats,repo["RID"]) - userRepoCount=userRepoCount+1 - # For debugging: - # print repo["USERNAME"]+" "+ repo["RNAME"] - - # update global repo counter - processedRepos=processedRepos+1 - # fetch next repository - repo=ibm_db.fetch_assoc(reposStmt) - except: - errortext=errortext+str(repo["RID"])+" " - # fetch next repository - repo=ibm_db.fetch_assoc(reposStmt) - # insert log entry - ts = time.gmtime() - logtext=logtext+str(processedRepos)+"/"+str(repoCount)+")" - if errortext !="": - logtext=logtext+", repo errors: "+errortext - res=ibm_db.execute(logStmt,(tenant["TID"],time.strftime("%Y-%m-%d %H:%M:%S", ts),userRepoCount,logtext)) - # fetch next system user - tenant=ibm_db.fetch_assoc(allTenants) - return {"repoCount": repoCount} - -if __name__ == "__main__": - main(json.loads(sys.argv[1])) diff --git a/functions/ghstats.zip b/functions/ghstats.zip deleted file mode 100644 index 71b1cb5..0000000 Binary files a/functions/ghstats.zip and /dev/null differ diff --git a/functions/setup.sh b/functions/setup.sh deleted file mode 100755 index 1ea44c2..0000000 --- a/functions/setup.sh +++ /dev/null @@ -1,32 +0,0 @@ -#!/bin/bash - -# Automatically set up services and actions for tutorial on -# regular Github statistics -# -# Written by Henrik Loeser - -# We need to pull down the githubpy file before packaging, -# then create the zip file and thereafter delete githubpy again (or leave it?). -# -# Fetch the github module: -# wget https://raw.githubusercontent.com/michaelliao/githubpy/master/github.py -# -# Pack the action code and the github module into a zip archive -# zip -r ghstats.zip __main__.py github.py -# -# Ok, now we can deploy the objects - -# service names -. ./../servicenames.sh - -# Create the action to collect statistics -ibmcloud fn action create collectStats --kind python-jessie:3 ghstats.zip - -# Bind the service credentials to the action -ibmcloud fn service bind dashDB collectStats --instance $DB_service --keyname $DB_service_key - -# Create a trigger for firing off daily at 6am -ibmcloud fn trigger create myDaily --feed /whisk.system/alarms/alarm --param cron "0 6 * * *" --param startDate "2018-03-21T00:00:00.000Z" --param stopDate "2018-12-31T00:00:00.000Z" - -# Create a rule to connect the trigger with the action -ibmcloud fn rule create myStatsRule myDaily collectStats diff --git a/servicenames.sh b/servicenames.sh deleted file mode 100644 index 4ac4d71..0000000 --- a/servicenames.sh +++ /dev/null @@ -1,15 +0,0 @@ -# Define the service names used across the scripts. -# When updating here, please also update in backend/manifest.yml - -AppID_service=ghstatsAppID -DB_service=ghstatsDB -DDE_service=ghstatsDDE - -# Initial name of service keys -DB_service_key=ghstatskey - -# Cloud Foundry application name - needed to rotate the credentials -CFApp_name=github-traffic-stats - -# Data center into which to deploy Db2 -Datacenter="eu-de:frankfurt" \ No newline at end of file diff --git a/setup.sh b/setup.sh deleted file mode 100755 index 173bc54..0000000 --- a/setup.sh +++ /dev/null @@ -1,16 +0,0 @@ -#!/bin/bash -# Script for simplified setup - -# service names -. ./servicenames.sh - -# Create a Db2 Warehouse service and a service key -ibmcloud cf create-service dashDB "Flex One" $DB_service -c '{"datacenter" : '${Datacenter}', "oracle_compatible":"no"}' -ibmcloud cf create-service-key ghstatsDB $DB_service_key - -# Create AppID service using "bx resource" command. AppID is available with -# resource groups. -ibmcloud resource service-instance-create $AppID_service appid graduated-tier us-south - -# Create DDE service (dynamic dashboard embedded) -ibmcloud resource service-instance-create $DDE_service dynamic-dashboard-embedded lite us-south