diff --git a/backend/database.sql b/backend/database.sql index 589026e..a2b56e0 100644 --- a/backend/database.sql +++ b/backend/database.sql @@ -31,7 +31,8 @@ create table repotraffic cuniques int not null default 0 ) organize by row; ---- index to improve tdate-based search +--- indexes to improve tdate-based search +create index repotraffic_ix_rid_tdate on repotraffic(rid,tdate); create index repotraffic_ix_tdate on repotraffic(tdate); --- The tenant, i.e. the user which accesses Github. diff --git a/backend/ghstats.py b/backend/ghstats.py index b307c2d..9c7b3b4 100644 --- a/backend/ghstats.py +++ b/backend/ghstats.py @@ -1,3 +1,16 @@ +# Manage repositories to automatically collect Github traffic statistics. +# Traffic data can be displayed, repositories added or deleted. +# +# Most actions are protected by using IBM Cloud App ID as an openID Connect +# authorization provider. Data is stored in a Db2 Warehouse on Cloud database. +# The app is designed to be ready for multi-tenant use, but not all functionality +# has been implemented yet. Right now, single-tenant operations are assumed. +# +# For the database schema see the file database.sql +# +# Written by Henrik Loeser (data-henrik), hloeser@de.ibm.com +# (C) 2018 by IBM + import flask, os, json, datetime, re, requests import github from flask import Flask, jsonify,redirect,request,render_template, url_for, Response, stream_with_context @@ -11,12 +24,17 @@ app = Flask(__name__) +# Check if we are in a Cloud Foundry environment, i.e., on IBM Cloud if 'VCAP_SERVICES' in os.environ: vcapEnv=json.loads(os.environ['VCAP_SERVICES']) + + # Configure access to Db2 Warehouse database dbInfo=vcapEnv['dashDB'][0] dbURI = dbInfo["credentials"]["uri"] app.config['SQLALCHEMY_DATABASE_URI']=dbURI app.config['SQLALCHEMY_TRACK_MODIFICATIONS']=False + + # Configure access to App ID service and openID Connect client appIDInfo = vcapEnv['AppID'][0] provider_config={ "issuer": "appid-oauth.ng.bluemix.net", @@ -29,6 +47,10 @@ "client_id": appIDInfo['credentials']['clientId'], "client_secret": appIDInfo['credentials']['secret'] } + + # Configure access to DDE service + DDE=vcapEnv['dynamic-dashboard-embedded'][0]['credentials'] + # See http://flask.pocoo.org/docs/0.12/config/ app.config.update({'SERVER_NAME': json.loads(os.environ['VCAP_APPLICATION'])['uris'][0], 'SECRET_KEY': 'my_secret_key', @@ -50,45 +72,14 @@ 'PERMANENT_SESSION_LIFETIME': 2592000, # session time in seconds (30 days) 'DEBUG': True}) +# Initialize openID Connect client auth = OIDCAuthentication(app, provider_configuration_info=provider_config, client_registration_info=client_info,userinfo_endpoint_method=None) - +# Initialize SQLAlchemy for our database db = SQLAlchemy(app, session_options={'autocommit': True}) -class User(db.Model): - __tablename__ = 'tenants' - tid = Column(Integer, primary_key=True) - fname = Column(String) - lname = Column(String) - email = Column(String) - ghuser = Column(String) - ghtoken = Column(String) - -class Repo(db.Model): - __tablename__ = 'repos' - rid = Column(Integer, primary_key=True) - rname = Column(String) - ghserverid = Column(Integer) - oid = Column(Integer) - schedule = Column(Integer) - - @property - def serialize(self): - return { - 'id' : self.rid, - 'rname' : self.rname, - 'oid' : self.oid - } - -class AdminUser(db.Model): - __tablename__ = 'adminusers' - aid = Column(Integer, primary_key=True) - auser = Column(String) - email = Column(String) - - - +# Encoder to handle some raw data correctly def alchemyencoder(obj): """JSON encoder function for SQLAlchemy special classes.""" if isinstance(obj, datetime.date): @@ -96,6 +87,7 @@ def alchemyencoder(obj): elif isinstance(obj, decimal.Decimal): return float(obj) +# Set the role for the current session user def setuserrole(email=None): result = db.engine.execute("select role from adminroles ar, adminusers au where ar.aid=au.aid and au.email=?",email) for row in result: @@ -106,7 +98,6 @@ def setuserrole(email=None): flask.session['userrole']=None return None - # Has the user the role of administrator? def isAdministrator(): return bool(flask.session['userrole'] & 1) @@ -154,7 +145,6 @@ def firststep(): @app.route('/admin/secondstep', methods=['POST']) @auth.oidc_auth def secondstep(): - print flask.session['id_token']['email'] username=request.form['username'] ghuser=request.form['ghuser'] ghtoken=request.form['ghtoken'] @@ -183,51 +173,39 @@ def secondstep(): trans.commit() except: trans.rollback() - # for now raise error, but ideally report error and return to welcome page - raise - return jsonify(stmts=dbstatements) - -@app.route('/repos/dashboard') -@auth.oidc_auth -def dashboard(): - # print request.url_root - #body={ "expiresIn": 3600, "webDomain" : "https://myportal.mybluemix.net" } - body={ "expiresIn": 3600, "webDomain" : request.url_root } - # print body - ddeUri=DDE['api_endpoint_url']+'v1/session' - res = requests.post(ddeUri, data=json.dumps(body) , auth=(DDE['client_id'], DDE['client_secret']), headers={'Content-Type': 'application/json'}) - # print res.text - # print json.loads(res.text)['sessionId'] - return render_template('dashboard.html',sessionInfo=json.loads(res.text)) - - + # for now ignore error and return to index page, but ideally report error and return to welcome page + return redirect(url_for('index')) + return redirect(url_for('listrepos')) +# Official login URI, redirects to repo stats after processing @app.route('/login') @auth.oidc_auth def login(): setuserrole(flask.session['id_token']['email']) - return render_template('welcome.html',id=flask.session['id_token'], role=flask.session['userrole']) + return redirect(url_for('repostatistics')) -@app.route('/welcome') +# Show a user profile +@app.route('/user') +@app.route('/user/profile') @auth.oidc_auth -def index2(): - return "Welcome, "+flask.session['id_token']['email'] - +def profile(): + return render_template('profile.html',id=flask.session['id_token'], role=flask.session['userrole']) +# End the session by logging off @app.route('/logout') @auth.oidc_logout def logout(): flask.session['userrole']=None return redirect(url_for('index')) -@app.route('/admin/newuser') +# Form to enter new tenant data +@app.route('/admin/newtenant') @auth.oidc_auth -def newuser(): +def newtenant(): return render_template('newuser.html') - - +# Show table with system logs @app.route('/admin/systemlog') @auth.oidc_auth def systemlog(): @@ -235,9 +213,23 @@ def systemlog(): result = db.engine.execute("select tid, completed, numrepos, state from systemlog where completed >(current date - 21 days) order by completed desc, tid asc") return render_template('systemlog.html',logs=result) else: - return "no logs" # should go to error or info page + return render_template('notavailable.html') # should go to error or info page + +# return page with the repository stats +@app.route('/repos/stats') +@auth.oidc_auth +def repostatistics(): + statstmt="""select r.rid,r.orgname,r.reponame,r.tdate,r.viewcount,r.vuniques,r.clonecount,r.cuniques + from v_repostats r, v_adminuserrepos v + where r.rid=v.rid + and v.email=? """ + # IDEA: expand to limit number of selected days, e.g., past 30 days + result = db.engine.execute(statstmt,flask.session['id_token']['email']) + return render_template('repostats.html',stats=result) +# Show list of managed repositories +@app.route('/repos') @app.route('/repos/list') @auth.oidc_auth def listrepos(): @@ -245,17 +237,16 @@ def listrepos(): result = db.engine.execute("select rid,orgname, reponame from v_adminrepolist where email=? order by rid asc",flask.session['id_token']['email']) return render_template('repolist.html',repos=result) else: - return "no repos" # should go to error or info page + return render_template('notavailable.html') # should go to error or info page -@app.route('/repos/newrepo', methods=['POST']) +# Process the request to add a new repository +@app.route('/api/newrepo', methods=['POST']) @auth.oidc_auth def newrepo(): if isTenant(): # Access form data from app orgname=request.form['orgname'] reponame=request.form['reponame'] - # Log to stdout stream - print orgname # could check if repo exists # but skipping to reduce complexity @@ -267,18 +258,18 @@ def newrepo(): aid=None rid=None orgid=None - ghstmt="""select atrr.tid, au.aid,u.ghuser,u.ghtoken + ghstmt="""select atrr.tid, au.aid,t.ghuser,t.ghtoken from admintenantreporoles atrr, adminusers au, adminroles ar, tenants t where ar.aid=au.aid and atrr.aid=au.aid and t.tid=atrr.tid - and bitand(aurr.role,4)>0 + and bitand(atrr.role,4)>0 and au.email=? """ githubinfo = connection.execute(ghstmt,flask.session['id_token']['email']) for row in githubinfo: tid=row['tid'] aid=row['aid'] - orgidinfo = connection.execute("select tid from ghorgusers where username=?",orgname) + orgidinfo = connection.execute("select oid from ghorgusers where username=?",orgname) for row in orgidinfo: orgid=row['oid'] if orgid is None: @@ -293,18 +284,21 @@ def newrepo(): except: trans.rollback() raise + # Log to stdout stream + print "Created repo with id "+str(rid) return jsonify(message="Your new repo ID: "+str(rid), repoid=rid) else: return jsonify(message="Error: no repository added") # should go to error or info page -@app.route('/repos/deleterepo', methods=['POST']) +# Process the request to delete a repository +@app.route('/api/deleterepo', methods=['POST']) @auth.oidc_auth def deleterepo(): if isTenant(): # Access form data from app repoid=request.form['repoid'] # Log to stdout stream - print repoid + print "Deleted repo with id "+str(repoid) # could check if repo exists # but skipping to reduce complexity @@ -334,40 +328,18 @@ def deleterepo(): - -@app.route('/datatest') -@auth.oidc_auth -def datatest(): - return render_template('data.html', - items=User.query.all() ) - -@app.route('/datatest2') -@auth.oidc_auth -def datatest2(): - return render_template('data2.html', - reposs=Repo.query.all() ) - -@app.route('/datatest3') -@auth.oidc_auth -def datatest3(): - return jsonify(json_list=[i.serialize for i in Repo.query.all()]) - -@app.route('/repos/stats') -@auth.oidc_auth -def repostatistics(): - statstmt="""select r.rid,r.orgname,r.reponame,r.tdate,r.viewcount,r.vuniques,r.clonecount,r.cuniques - from v_repostats r, v_adminuserrepos v - where r.rid=v.rid - and v.email=? """ - # expand to limit number of selected days, e.g., past 30 days - result = db.engine.execute(statstmt,flask.session['id_token']['email']) - return render_template('repostats.html',stats=result) - -@app.route('/datatest5') +@app.route('/repos/dashboard') @auth.oidc_auth -def datatest5(): - result = db.engine.execute("select r.rid,r.orgname,r.reponame,r.tdate,r.viewcount from v_repostats r, v_adminuserrepos v where v.email=? and r.rid=v.rid",flask.session['id_token']['email']) - return json.dumps([dict(r) for r in result],default=alchemyencoder) +def dashboard(): + # print request.url_root + #body={ "expiresIn": 3600, "webDomain" : "https://myportal.mybluemix.net" } + body={ "expiresIn": 3600, "webDomain" : request.url_root } + # print body + ddeUri=DDE['api_endpoint_url']+'v1/session' + res = requests.post(ddeUri, data=json.dumps(body) , auth=(DDE['client_id'], DDE['client_secret']), headers={'Content-Type': 'application/json'}) + # print res.text + # print json.loads(res.text)['sessionId'] + return render_template('dashboard.html',sessionInfo=json.loads(res.text)) # return the currently active user as csv file @@ -393,6 +365,14 @@ def generate(): yield ','.join(map(str,row)) + '\n' return Response(stream_with_context(generate()), mimetype='text/csv') + +@app.route('/admin') +@app.route('/data') +@auth.oidc_auth +def not_available(): + return render_template('notavailable.html') + +# error function for auth module @auth.error_view def error(error=None, error_description=None): return jsonify({'error': error, 'message': error_description}) diff --git a/backend/static/ghstats.js b/backend/static/ghstats.js index a692577..235fd7b 100644 --- a/backend/static/ghstats.js +++ b/backend/static/ghstats.js @@ -15,7 +15,7 @@ function addRepo() { document.getElementById("reponame").value = ''; } }; - xhttp.open('POST', "/repos/newrepo"); + xhttp.open('POST', "/api/newrepo"); xhttp.setRequestHeader("Content-type", "application/x-www-form-urlencoded"); var postVars = 'orgname=' + orgname + '&reponame=' + reponame; xhttp.send(postVars); @@ -36,7 +36,7 @@ function deleteRepo() { document.getElementById("repoid").value = ''; } }; - xhttp.open('POST', "/repos/deleterepo"); + xhttp.open('POST', "/api/deleterepo"); xhttp.setRequestHeader("Content-type", "application/x-www-form-urlencoded"); var postVars = 'repoid=' + repoid; xhttp.send(postVars); diff --git a/backend/templates/data.html b/backend/templates/data.html deleted file mode 100644 index 602d64d..0000000 --- a/backend/templates/data.html +++ /dev/null @@ -1,25 +0,0 @@ -{% extends "layout.html" %} -{% block body %} -
-

Repo Owners

- - - -
# - Firstname - Lastname - Email - Github user name - {%- for item in items %} -
{{ item.uid }} - {{ item.fname }} - {{ item.lname }} - {{ item.email }} - {{ item.ghuser }} - {%- endfor %} -
-

- - -

-{% endblock %} diff --git a/backend/templates/data2.html b/backend/templates/data2.html deleted file mode 100644 index 03f5c8d..0000000 --- a/backend/templates/data2.html +++ /dev/null @@ -1,21 +0,0 @@ -{% extends "layout.html" %} -{% block body %} -
-

Repo Owners

- - - -
# - Repository - OwnerID - {%- for repo in repos %} -
{{ repo.rid }} - {{ repo.rname }} - {{ repo.ownerid }} - {%- endfor %} -
-

- - -

-{% endblock %} diff --git a/backend/templates/index.html b/backend/templates/index.html index 15cc79c..dafee5d 100644 --- a/backend/templates/index.html +++ b/backend/templates/index.html @@ -1,16 +1,17 @@ {% extends "layout.html" %} {% block title %} -Github Traffic - Welcome +Github Traffic Analytics {% endblock %} {% block body %}

Welcome to Github Traffic Analytics

+

- Login to continue to access to the traffic statistics. + Please login to continue to access to the traffic statistics.

- +
{% endblock %} diff --git a/backend/templates/layout.html b/backend/templates/layout.html index b912c1a..a1b20c5 100644 --- a/backend/templates/layout.html +++ b/backend/templates/layout.html @@ -10,7 +10,76 @@ - + + +
{% block body %}{% endblock %} @@ -20,10 +89,37 @@
+ + {% block extra_javascripts %} {% endblock %} + + + diff --git a/backend/templates/newuser.html b/backend/templates/newuser.html index e594553..a2482d5 100644 --- a/backend/templates/newuser.html +++ b/backend/templates/newuser.html @@ -2,10 +2,10 @@ {% block body %}
-

Add new user

+

Add new tenant

-
+
@@ -14,16 +14,16 @@

Add new user

- +
- +
- + @@ -33,7 +33,7 @@

Add new user

- +
diff --git a/backend/templates/notavailable.html b/backend/templates/notavailable.html new file mode 100644 index 0000000..1b338ec --- /dev/null +++ b/backend/templates/notavailable.html @@ -0,0 +1,11 @@ +{% extends "layout.html" %} +{% block title %} +Github Traffic Analytics - Not available yet +{% endblock %} + + +{% block body %} +
+ Thank you for your interested, but this is not available yet. +
+{% endblock %} diff --git a/backend/templates/profile.html b/backend/templates/profile.html new file mode 100644 index 0000000..cf1e2eb --- /dev/null +++ b/backend/templates/profile.html @@ -0,0 +1,44 @@ +{% extends "layout.html" %} +{% block title %} +Github Traffic Analytics - Profile +{% endblock %} + + +{% block body %} +
+

User profile

+
+ +
+ + {% if role is divisibleby(2) %} +

sysmaint

+ {% endif %} +
+{% endblock %}