scitran · gsfr · Nov 14, 2017 · Nov 17, 2017 · Nov 17, 2017 · Nov 17, 2017
diff --git a/.dockerignore b/.dockerignore
@@ -1,15 +1,8 @@
 .git
+**/.DS_Store
+**/.*sw[op]
+**/*.py[co]
+**/*.egg-info
 
-# --- Below Should mirror .gitignore, without leading '/' --- #
-.DS_Store
-.*sw[op]
-*.py[co]
-*.egg-info
-persistent
-runtime
-bootstrap.json
-.cache
-.coverage*
-coverage.xml
-htmlcov
-node_modules
+/.coverage*
+/htmlcov
diff --git a/CONTRIBUTING.md → .github/CONTRIBUTING.md b/CONTRIBUTING.md → .github/CONTRIBUTING.md
diff --git a/PULL_REQUEST_TEMPLATE.md → .github/PULL_REQUEST_TEMPLATE.md b/PULL_REQUEST_TEMPLATE.md → .github/PULL_REQUEST_TEMPLATE.md
diff --git a/.gitignore b/.gitignore
@@ -2,12 +2,6 @@
 .*sw[op]
 *.py[co]
 *.egg-info
-/persistent
-/runtime
-bootstrap.json
-.cache
+
 /.coverage*
-coverage.xml
 /htmlcov
-node_modules/
-/bin/accesslog.csv
diff --git a/.travis.yml b/.travis.yml
@@ -1,26 +1,40 @@
-sudo: required
 dist: trusty
+sudo: required
 
 services:
-  - mongodb
+    - docker
+
 env:
-  global:
-    secure: HELJx6WPr+W2S0FV47KkRdlS9NCqlMcdRMK8xWgRqqrEPv24KEvNnHxCy0tRbzITqadYtyvI1MtqtmpG04uty8Gpkc7w6L6LMJ/OuLG0gVX7AnaUovYTlY04m1/L9oyzOrTDXk5J/BKbcyiz7uJtkTc/A8MMZAFfZh7hmhLID78= # BUILD_TRIGGER_URL
+    global:
+        - DOCKER_DIR="$HOME/.cache/docker"
+        - secure: "HhT1TdJcpqys8juVMw/DIZeK7oD4595TEKH5KlowH7MvwwFAUyQFb5W63F8dgk7elvRG+3fmga/m1JfXO+Iu7PVD912eiNDagW9aB3CEl3Z8zg+JUL8IjpMCkyKQDyJMnfOkrzdxdaqfOK+WmF+13f2qBu9Kc7wdXuzgHQrg4+0=" # CI_REGISTRY_USER
+        - secure: "hh7VDZnkxgl/vqHtS4IpXfIAckKpVQvoCzNW7fstr5Mcu8KNiCWIPgObBRm+m13aqpcFTMWQ6lT2kzORz2wWRbDeVhI1eGWOJswGNHPHZLO0Jaei6yfY2nY2mpxZbl+vdg00jkN64mi1ab3e++QgeLFruW0gyNefXX7E5L/mHTs=" # CI_REGISTRY_PASS
+
+cache:
+    directories:
+        - $DOCKER_DIR
+
+before_install:
+    - sudo apt-get update
+    - sudo apt-get -y install docker-ce realpath
+    - docker login -u $CI_REGISTRY_USER -p $CI_REGISTRY_PASS
 
-install:
-  - bin/install-ubuntu.sh
-  - tests/bin/setup-integration-tests-ubuntu.sh
+install: true
 
 script:
-  - SCITRAN_PERSISTENT_DB_PORT=27017 tests/bin/run-tests-ubuntu.sh
+    - test -f "$DOCKER_DIR/image.tar" && docker load -i "$DOCKER_DIR/image.tar" || true
+    - docker build -t core:build --target build .
+    - docker build -t core:dist --target dist --build-arg VCS_BRANCH="$TRAVIS_BRANCH" --build-arg VCS_COMMIT="$TRAVIS_COMMIT" .
+    - docker build -t core:dev --target dev .
+    - docker save -o "$DOCKER_DIR/image.tar" $(docker history -q core:build | grep -v '<missing>') $(docker history -q core:dist | grep -v '<missing>')
+    - ./tests/bin/docker-tests.sh --image core:dev
 
 after_success:
-  - if [ "$TRAVIS_BRANCH" == "master" -o  "$TRAVIS_EVENT_TYPE" == "pull_request" ]; then
-        bash <(curl -s https://codecov.io/bash) -cF python;
-    fi
-  - if [ "$TRAVIS_TAG" ]; then
-        ./docker/build-trigger.sh Tag "$TRAVIS_TAG" "$BUILD_TRIGGER_URL";
-    fi
-  - if [ "$TRAVIS_EVENT_TYPE" == "push" -a "$TRAVIS_BRANCH" == "master" ]; then
-        ./docker/build-trigger.sh Branch "$TRAVIS_BRANCH" "$BUILD_TRIGGER_URL";
-    fi
+    - if [ "$TRAVIS_TAG" ]; then
+          docker tag core:dist scitran/core:$TRAVIS_TAG;
+          docker push scitran/core:$TRAVIS_TAG;
+      fi
+    - if [ "$TRAVIS_EVENT_TYPE" == "push" -a "$TRAVIS_BRANCH" == "nginx-unit" ]; then
+          docker tag core:dev scitran/core:nginx-unit;
+          docker push scitran/core:nginx-unit;
+      fi
diff --git a/Dockerfile b/Dockerfile
@@ -1,103 +1,51 @@
-#
-# Image used for hosting scitran core with uwsgi.
-#
-# Example usage is in README.md
-#
-
-FROM ubuntu:14.04
-
-
-# Install pre-requisites
-RUN apt-get update \
-	&& apt-get install -y \
-		build-essential \
-		ca-certificates curl \
-		libatlas3-base \
-		numactl \
-		python-dev \
-		python-pip \
-		libffi-dev \
-		libssl-dev \
-		libpcre3 \
-		libpcre3-dev \
-		git \
-	&& rm -rf /var/lib/apt/lists/* \
-	&& pip install -U pip
-
-
-# Grab gosu for easy step-down from root in a docker-friendly manner
-# https://github.com/tianon/gosu
-#
-# Alternate key servers are due to reliability issues with ha.pool.sks-keyservers.net
-RUN curl -o /usr/local/bin/gosu -SL "https://github.com/tianon/gosu/releases/download/1.6/gosu-$(dpkg --print-architecture)" \
-	&& curl -o /tmp/gosu.asc -SL "https://github.com/tianon/gosu/releases/download/1.6/gosu-$(dpkg --print-architecture).asc" \
-	&& export GNUPGHOME="$(mktemp -d)" \
-	&& for server in $(shuf -e ha.pool.sks-keyservers.net \
-	                            hkp://p80.pool.sks-keyservers.net:80 \
-	                            keyserver.ubuntu.com \
-	                            hkp://keyserver.ubuntu.com:80 \
-	                            pgp.mit.edu) ; do \
-	        gpg --keyserver "$server" --recv-keys B42F6819007F00F88E364FD4036A9C25BF357DD4 && break || : ; \
-	    done \
-	&& gpg --batch --verify /tmp/gosu.asc /usr/local/bin/gosu \
-	&& rm -r "$GNUPGHOME" /tmp/gosu.asc \
-	&& chmod +x /usr/local/bin/gosu
-
-
-# Setup environment
-WORKDIR /var/scitran
-
-RUN mkdir -p \
-      /var/scitran/config \
-      /var/scitran/data \
-      /var/scitran/code/api \
-      /var/scitran/logs \
-      /var/scitran/keys
-
-# Declaring a volume makes the intent to map externally explicit. This enables
-# the contents to survive/persist across container versions, and easy access
-# to the contents outside the container.
-#
-# Declaring the VOLUME in the Dockerfile guarantees the contents are empty
-# for any new container that doesn't specify a volume map via 'docker run -v '
-# or similar option.
-#
-VOLUME /var/scitran/keys
-VOLUME /var/scitran/data
-VOLUME /var/scitran/logs
-
-
-# Install pip modules
-#
-# Split this out for better cache re-use.
-#
-COPY requirements.txt docker/requirements-docker.txt /var/scitran/code/api/
-
-RUN pip install --upgrade pip wheel setuptools \
-  && pip install -r /var/scitran/code/api/requirements-docker.txt \
-  && pip install -r /var/scitran/code/api/requirements.txt
-
-COPY tests /var/scitran/code/api/tests/
-RUN bash -e -x /var/scitran/code/api/tests/bin/setup-integration-tests-ubuntu.sh
-
-
-# Copy full repo
-#
-COPY . /var/scitran/code/api/
-
-COPY docker/uwsgi-entrypoint.sh /var/scitran/
-COPY docker/uwsgi-config.ini /var/scitran/config/
-
-
-
-# Inject build information into image so the source of the container can be
-# determined from within it.
-ARG BRANCH_LABEL=NULL
-ARG COMMIT_HASH=0
-COPY docker/inject_build_info.sh /
-RUN /inject_build_info.sh ${BRANCH_LABEL} ${COMMIT_HASH} \
-  && rm /inject_build_info.sh
-
-
-ENTRYPOINT ["/var/scitran/uwsgi-entrypoint.sh"]
-CMD ["uwsgi", "--ini", "/var/scitran/config/uwsgi-config.ini", "--http", "0.0.0.0:8080", "--http-keepalive", "--so-keepalive", "--add-header", "Connection: Keep-Alive" ]
+FROM python:2.7-alpine3.7 as build
+
+RUN apk --no-cache add build-base curl
+
+WORKDIR /src/nginx-unit
+
+RUN curl -L https://github.com/nginx/unit/archive/0.4.tar.gz | tar xz --strip-components 1
+RUN ./configure --prefix=/usr/local --modules=lib --state=/var/local/unit --pid=/var/unit.pid --log=/var/log/unit.log \
+ && ./configure python \
+ && make install
+
+
+FROM python:2.7-alpine3.7 as dist
+
+RUN apk --no-cache add git
+
+COPY --from=build /usr/local/sbin/unitd /usr/local/sbin/unitd
+COPY --from=build /usr/local/lib/python.unit.so /usr/local/lib/python.unit.so
+
+EXPOSE 80 8088
+VOLUME /data/db /data/persistent
+
+WORKDIR /src/core
+ENV SCITRAN_PERSISTENT_DATA_PATH=/data/persistent
+
+COPY nginx-unit.json /var/local/unit/conf.json
+COPY requirements.txt requirements.txt
+RUN pip install -r requirements.txt
+
+COPY . .
+RUN pip install -e .
+
+ARG VCS_BRANCH=NULL
+ARG VCS_COMMIT=NULL
+RUN ./bin/build_info.sh $VCS_BRANCH $VCS_COMMIT | tee /version.json
+
+CMD ["unitd", "--control", "*:8088", "--no-daemon", "--log", "/dev/stdout"]
+
+
+FROM dist as dev
+
+EXPOSE 27017
+
+RUN apk --no-cache add mongodb nginx
+RUN mkdir /run/nginx
+
+COPY nginx.conf /etc/nginx/nginx.conf
+
+RUN pip install -r tests/requirements.txt
+
+CMD ["./bin/dev+mongo.sh"]
diff --git a/README.md b/README.md
@@ -19,23 +19,6 @@ SciTran Core is a RESTful HTTP API, written in Python and backed by MongoDB. It
 
 
 ### Usage
-**Currently Python 2 Only**
-
-#### OSX
-```
-$ ./bin/run-dev-osx.sh --help
-```
-
-For the best experience, please upgrade to a recent version of bash.
-```
-brew install bash bash-completion
-sudo dscl . -create /Users/$(whoami) UserShell /usr/local/bin/bash
-```
-
-#### Ubuntu
 ```
-mkvirtualenv scitran-core
-./bin/install-ubuntu.sh
-uwsgi --http :8080 --master --wsgi-file bin/api.wsgi -H $VIRTUAL_ENV \
-    --env SCITRAN_PERSISTENT_DB_URI="mongodb://localhost:27017/scitran-core"
+docker run -p 80:80 -e SCITRAN_CORE_DRONE_SECRET=secret scitran/core
 ```
diff --git a/TESTING.md b/TESTING.md
diff --git a/api/app.py b/api/app.py
@@ -0,0 +1,3 @@
+from .web import start
+
+application = start.app_factory()
diff --git a/api/web/start.py b/api/web/start.py
@@ -16,7 +16,7 @@ def save_coverage(cov):
     def start_coverage():
         import coverage
         print("Enabling code coverage")
-        cov = coverage.coverage(source=["api"], data_suffix="integration-tests")
+        cov = coverage.coverage(source=["api"], data_file="/tmp/.coverage.integration-tests")
         cov.start()
         atexit.register(save_coverage, cov)
 

diff --git a/bin/api.wsgi b/bin/api.wsgi
diff --git a/bin/build_info.sh b/bin/build_info.sh
@@ -0,0 +1,11 @@
+#!/usr/bin/env sh
+
+set -eu
+
+cat <<EOF
+{
+  "branch": "$1",
+  "commit": "$2",
+  "timestamp": "$(date -u +"%Y-%m-%dT%H:%M:%SZ")"
+}
+EOF
diff --git a/bin/oneoffs/cas_statistic.py → bin/cas_statistic.py b/bin/oneoffs/cas_statistic.py → bin/cas_statistic.py
@@ -1,3 +1,5 @@
+#!/usr/bin/env python
+
 import os
 import pymongo
 from collections import Counter

diff --git a/bin/database.py → bin/db_upgrade.py b/bin/database.py → bin/db_upgrade.py
diff --git a/bin/dev+mongo.sh b/bin/dev+mongo.sh
@@ -0,0 +1,16 @@
+#!/usr/bin/env sh
+
+unset CDPATH
+cd "$(dirname "$0")"
+
+nginx
+
+mongod &
+
+if [ "$(stat -c %U:%G $SCITRAN_PERSISTENT_DATA_PATH)" != "nobody:nobody" ]; then
+    chown nobody:nobody -R $SCITRAN_PERSISTENT_DATA_PATH
+fi
+
+./db_upgrade.py upgrade_schema
+
+exec unitd --control "*:8088" --no-daemon --log /dev/stdout
Original file line number	Diff line number	Diff line change
		@@ -0,0 +1,3 @@
		from .web import start

		application = start.app_factory()