Merge branch 'release/v17.4.27'

.catalyze/post-build

@@ -0,0 +1,5 @@
+#!/bin/sh -e
+# Datica post-build hook script
+# https://resources.datica.com/compliant-cloud/articles/buildpacks-custom/#pre-and-post-build-hooks
+
+FLASK_APP=manage.py flask sync

.travis.yml

@@ -13,7 +13,6 @@ env:
     - TOXENV=py27
   global:
     - SQLALCHEMY_DATABASE_TEST_URI='postgresql://postgres:@localhost/portal_unit_tests'
-    - LOG_FOLDER='/tmp/shared_service_log'
     - PACKAGE_NAME=${PACKAGE_NAME:-true_nth_usa_portal}
     - IMAGE_NAME=${IMAGE_NAME:-portal_web}
     - DEB_REPO=${DEB_REPO:-portal-deb}

app.json

@@ -7,10 +7,6 @@
   "logo": "https://github.com/uwcirg/true_nth_usa_portal/raw/develop/portal/static/img/logo.png",
   "success_url": "/about",
   "env": {
-    "LOG_FOLDER": {
-      "description": "Temporary log directory override; should write to stdout if unconfigured",
-      "value": "/tmp/logs"
-    },
     "SERVER_NAME": {
       "description": "Fully qualified domain name of system, excluding protocol (HTTP/S). See App Name",
       "value": "HEROKU_APP_NAME.herokuapp.com"
@@ -30,6 +26,6 @@
     }
   ],
   "scripts": {
-    "postdeploy": "python manage.py initdb"
+    "postdeploy": "FLASK_APP=manage.py flask sync"
   }
 }

bin/deploy.sh

@@ -7,7 +7,6 @@ usage() {
     echo -e "-s\n 'seed' the database" >&2
     echo -e "-v\n Be verbose" >&2
     echo -e "-f\n Force all conditional deployment processes" >&2
-    echo -e "-i\n Run initial deployment procedure" >&2
     exit 1
 }
 
@@ -48,9 +47,6 @@ while getopts ":b:p:ivsf" option; do
         v)
             VERBOSE=true
             ;;
-        i)
-            INIT=true
-            ;;
         s)
             SEED=true
             ;;
@@ -71,6 +67,7 @@ fi
 
 export GIT_WORK_TREE="$repo_path"
 export GIT_DIR="${GIT_WORK_TREE}/.git"
+export FLASK_APP="${GIT_WORK_TREE}/manage.py"
 
 if [[ -z $BRANCH ]]; then
     BRANCH="develop"
@@ -107,28 +104,21 @@ fi
 
 # DB Changes
 if [[
-    ($FORCE || ( -n $(git diff $old_head $new_head -- ${GIT_WORK_TREE}/portal/migrations) && $? -eq 0 )) &&
-    -z $INIT
+    ($FORCE || ( -n $(git diff $old_head $new_head -- ${GIT_WORK_TREE}/portal/migrations) && $? -eq 0 ))
 ]]; then
     activate_once
 
     if [[ $VERBOSE ]]; then
         echo "Running database migrations"
     fi
-    python "${GIT_WORK_TREE}/manage.py" db upgrade
+    flask sync
 fi
 
 # New seed data
 if [[ $FORCE || $SEED || ( -n $(git diff $old_head $new_head -- ${GIT_WORK_TREE}/portal/models) && $? -eq 0 ) ]]; then
     activate_once
-
-    if [[ $INIT ]]; then
-        echo "Initializing database"
-        python "${GIT_WORK_TREE}/manage.py" initdb
-    else
-        echo "Seeding database"
-        python "${GIT_WORK_TREE}/manage.py" seed
-    fi
+    echo "Seeding database"
+    flask seed
 fi
 
 

docker/Dockerfile

@@ -6,32 +6,10 @@ RUN \
     apt-get update && \
     apt-get dist-upgrade --yes && \
     apt-get install --yes --force-yes --no-install-recommends \
-        apache2 \
-        libapache2-mod-wsgi \
         libpq5 \
         python2.7 && \
     apt-get clean
 
-RUN printf '\n\
-Listen 0.0.0.0:${PORT}\n\
-ServerName ${SERVER_NAME}\n\
-ErrorLog /dev/stdout\n\
-<VirtualHost *:${PORT}>\n\
-  LogLevel info\n\
-  ErrorLog /dev/stdout\n\
-  CustomLog /dev/stdout combined\n\
-  WSGIPassAuthorization On\n\
-  WSGIDaemonProcess stg_cs user=${APACHE_RUN_USER} group=${APACHE_RUN_GROUP} threads=5\n\
-  WSGIScriptAlias / /var/www/portal.wsgi\n\
-  <Directory /var/www/>\n\
-    WSGIProcessGroup stg_cs\n\
-    WSGIApplicationGroup %%{GLOBAL}\n\
-    Require all granted\n\
-  </Directory>\n\
-</VirtualHost>\n\
-\n'\
-> /etc/apache2/sites-available/portal.conf
-
 ARG debian_repo="http://dl.bintray.com/v1/content/uwcirg/truenth"
 
 COPY artifacts /tmp/artifacts
@@ -51,39 +29,28 @@ RUN \
     apt-get clean && \
     rm --force --recursive --verbose /tmp/artifacts
 
-COPY portal.wsgi /var/www/
-
 ENV \
-    APACHE_RUN_USER="${APACHE_RUN_USER:-www-data}" \
-    LOG_FOLDER="/tmp/shared_service_log" \
-    PORT="${PORT:-8008}" \
+    RUN_USER="${RUN_USER:-www-data}" \
     PROJECT_DIR="/opt/venvs/portal"
 
 RUN \
-    mkdir -p ${LOG_FOLDER} ${PROJECT_DIR}/var/portal-instance/ && \
-    chown $APACHE_RUN_USER:$APACHE_RUN_USER \
-        /var/run/apache2/ \
-        ${LOG_FOLDER} \
-        ${PROJECT_DIR}/var/portal-instance/ && \
-    sed -i 's|Listen [[:digit:]]\+|Listen ${PORT}|g' /etc/apache2/ports.conf && \
-    a2ensite portal && a2dissite 000-default && a2disconf other-vhosts-access-log
+    mkdir --parents ${PROJECT_DIR}/var/portal-instance/ && \
+    chown $RUN_USER:$RUN_USER ${PROJECT_DIR}/var/portal-instance/
 
-USER ${APACHE_RUN_USER}
+USER ${RUN_USER}
 
 ENV \
     SERVER_NAME=${SERVER_NAME:-localhost} \
+    PORT="${PORT:-8008}" \
     FLASK_APP="${PROJECT_DIR}/bin/manage.py" \
     PERSISTENCE_FILE="${PERSISTENCE_FILE:-https://raw.githubusercontent.com/uwcirg/TrueNTH-USA-site-config/develop/site_persistence_file.json}"
 
 EXPOSE ${PORT}
 
 CMD \
-    . /etc/apache2/envvars && \
     . ${PROJECT_DIR}/bin/activate && \
     env && \
-    if [ -n "$(flask db current)" ]; then \
-        flask db migrate; \
-    else \
-        python ${FLASK_APP} initdb; \
-    fi && \
-    /usr/sbin/apache2 -D FOREGROUND
+    flask sync && \
+    gunicorn \
+        --bind "0.0.0.0:${PORT}" \
+    manage:app

manage.py

@@ -1,15 +1,15 @@
 """ Defines a series of scripts for running server and maintenance
 
-python manage.py --help
+FLASK_APP=manage.py flask --help
 
 """
 import os
+import click
 
-from flask_script import Manager
+import alembic.config
 from flask_migrate import Migrate, MigrateCommand
 
 from portal.app import create_app
-from portal.config import ConfigServer
 from portal.extensions import db
 from portal.models.i18n import upsert_to_template_file
 from portal.models.fhir import add_static_concepts
@@ -22,29 +22,73 @@
 from portal.site_persistence import SitePersistence
 
 app = create_app()
-manager = Manager(app)
-
-migrate = Migrate(
-    app,
-    db,
-    directory=os.path.join(app.root_path, 'migrations')
-)
-manager.add_command('db', MigrateCommand)
-manager.add_command('runserver', ConfigServer(host='0.0.0.0', threaded=True))
-
-
-@manager.command
-def initdb():
-    """Init/reset database."""
-    db.drop_all()
-    db.create_all()
+
+MIGRATIONS_DIR = os.path.join(app.root_path, 'migrations')
+migrate = Migrate(app, db, directory=MIGRATIONS_DIR)
+
+@app.cli.command()
+def runserver():
+    # Todo: figure out how to override default host in `flask run`
+    # http://click.pocoo.org/5/commands/#overriding-defaults
+    app.run(host='0.0.0.0')
+
+def _run_alembic_command(args):
+    """Helper to manage working directory and run given alembic commands"""
+    # Alembic looks for the alembic.ini file in CWD
+    # hop over there and then return to CWD
+    cwd = os.getcwd()
+    os.chdir(MIGRATIONS_DIR)
+
+    alembic.config.main(argv=args)
+
+    os.chdir(cwd)  # restore cwd
+
+
+def stamp_db():
+    """Run the alembic command to stamp the db with the current head"""
+    # if the alembic_version table exists, this db has been stamped,
+    # don't update to head, as it would potentially skip steps.
+    if db.engine.dialect.has_table(db.engine.connect(), 'alembic_version'):
+        return
+
+    _run_alembic_command(['--raiseerr', 'stamp', 'head'])
+
+
+def upgrade_db():
+    """Run any outstanding migration scripts"""
+    _run_alembic_command(['--raiseerr', 'upgrade', 'head'])
+
+
+@app.cli.command()
+def sync():
+    """Synchronize database with latest schema and persistence data.
+
+    Idempotent function takes necessary steps to build tables, migrate
+    schema and run `seed`.  Safe to run on existing or brand new databases.
+
+    To re/create the database, [delete and] create within the DBMS itself,
+    then invoke this function.
+    """
+    if not db.engine.dialect.has_table(db.engine.connect(), 'alembic_version'):
+        db.create_all()
+    stamp_db()
+    upgrade_db()
     seed(include_interventions=True)
 
 
-@manager.command
+@click.option('--include_interventions', '-i', default=False, help='Include (overwrite) intervention data')
+@click.option('--keep_unmentioned', '-k', default=False, help='Keep orgs and interventions not mentioned in persistence file')
+@app.cli.command(name="seed")
+def seed_command(include_interventions, keep_unmentioned):
+    seed(include_interventions, keep_unmentioned)
+
 def seed(include_interventions=False, keep_unmentioned=False):
     """Seed database with required data"""
-    add_static_concepts()
+
+    # Request context necessary for generating data from own HTTP APIs
+    with app.test_request_context():
+        add_static_concepts()
+
     add_static_interventions()
     add_static_organization()
     add_static_relationships()
@@ -60,7 +104,7 @@ def seed(include_interventions=False, keep_unmentioned=False):
     SitePersistence().import_(include_interventions, keep_unmentioned)
 
 
-@manager.command
+@app.cli.command()
 def export_site():
     """Generate JSON file containing dynamic site config
 
@@ -75,23 +119,20 @@ def export_site():
     SitePersistence().export()
 
 
-@manager.option('-u', '--username', dest='username')
+@click.option('--username', '-u', help='Username of user to purge.')
+@app.cli.command()
 def purge_user(username):
     """Purge the given user from the system"""
     permanently_delete_user(username)
 
 
-@manager.command
+@app.cli.command()
 def mark_test():
     """Designate all current users as test users"""
     flag_test()
 
 
-@manager.command
+@app.cli.command()
 def translations():
     """Add extracted DB strings to existing PO template file"""
     upsert_to_template_file()
-
-
-if __name__ == '__main__':
-    manager.run()

portal/app.py

@@ -161,8 +161,8 @@ def configure_logging(app):  # pragma: no cover
         sql_log.setLevel(logging.INFO)
         sql_log.addHandler(sql_file_handler)
 
-    if app.testing:
-        # Skip test mode. Just check standard output.
+    if app.testing or not app.config.get('LOG_FOLDER', None):
+        # Write logs to stdout by default and when testing
         return
 
     if not os.path.exists(app.config['LOG_FOLDER']):

portal/audit.py

@@ -55,9 +55,13 @@ def configure_audit_log(app):  # pragma: no cover
         return
 
     logging.addLevelName('AUDIT', AUDIT)
-    audit_log = os.path.join(app.config['LOG_FOLDER'], 'audit.log')
-    audit_file_handler = logging.FileHandler(audit_log, delay=True)
-    audit_file_handler.setLevel(AUDIT)
-    audit_file_handler.setFormatter(
+
+    audit_log_handler = logging.StreamHandler(sys.stdout)
+    if app.config.get('LOG_FOLDER', None):
+        audit_log = os.path.join(app.config['LOG_FOLDER'], 'audit.log')
+        audit_log_handler = logging.FileHandler(audit_log, delay=True)
+
+    audit_log_handler.setLevel(AUDIT)
+    audit_log_handler.setFormatter(
         logging.Formatter('%(asctime)s: %(message)s'))
-    app.logger.addHandler(audit_file_handler)
+    app.logger.addHandler(audit_log_handler)

portal/config.py

@@ -50,10 +50,7 @@ class BaseConfig(object):
     CELERY_RESULT_BACKEND = 'redis'
     DEBUG = False
     DEFAULT_MAIL_SENDER = '[email protected]'
-    LOG_FOLDER = os.environ.get(
-        'LOG_FOLDER',
-        os.path.join('/var/log', __package__)
-    )
+    LOG_FOLDER = os.environ.get('LOG_FOLDER', None)
     LOG_LEVEL = 'DEBUG'
 
     MAIL_USERNAME = '[email protected]'
@@ -125,33 +122,3 @@ class TestConfig(BaseConfig):
 
     WTF_CSRF_ENABLED = False
     FILE_UPLOAD_DIR = 'test_uploads'
-
-
-class ConfigServer(Server):  # pragma: no cover
-    """Correctly read Flask configuration values when running Flask
-
-    Flask-Script 2.0.5 does not read host and port specified in
-    SERVER_NAME.  This subclass fixes that.
-
-    Bug: https://github.com/smurfix/flask-script/blob/7dfaf2898d648761632dc5b3ba6654edff67ec57/flask_script/commands.py#L343
-
-    Values passed in when instance is called as a function override
-    those passed during initialization which override configured values
-
-    See https://github.com/smurfix/flask-script/issues/108
-    """
-    def __init__(self, port=None, host=None, **kwargs):
-        """Override default port and host
-
-        Allow fallback to configured values
-
-        """
-        super(ConfigServer, self).__init__(port=port, host=host, **kwargs)
-
-    def __call__(self, app=None, host=None, port=None, *args, **kwargs):
-        """Call app.run() with highest precedent configuration values"""
-        # Fallback to initialized value if None is passed
-        port = self.port if port is None else port
-        host = self.host if host is None else host
-        super(ConfigServer, self).__call__(app=app, host=host,
-                port=port, *args, **kwargs)