Docker for testing

.travis.yml

@@ -1,46 +1,59 @@
-language: python
-python:
-  - "2.7"
-  - "3.4"
-
-before_install:
-  # We do this conditionally because it saves us some downloading if the
-  # version is the same.
-  - if [[ "$TRAVIS_PYTHON_VERSION" == "2.7" ]]; then
-      wget https://repo.continuum.io/miniconda/Miniconda2-latest-Linux-x86_64.sh -O miniconda.sh;
-    else
-      wget https://repo.continuum.io/miniconda/Miniconda3-latest-Linux-x86_64.sh -O miniconda.sh;
-    fi
-  - bash miniconda.sh -b -p $HOME/miniconda
-  - export PATH="$HOME/miniconda/bin:$PATH"
-  - hash -r
-  - conda config --set always_yes yes --set changeps1 no
-  - conda update -q conda
-  # Useful for debugging any issues with conda
-  - conda info -a
-  # Replace dep1 dep2 ... with your dependencies
-  - conda install -c r r
-  - conda install -c r r-knitr
-install:
-  - pip install autopep8 pep8
-  - pip install .[all]
-
-# Cache isn't working yet because problems with expiration header. Disable for now because it's breaking builds.
-# cache: pip
-
-before_script:
-  - "pep8 --exclude knowledge_repo/app/migrations,build,deploy,kube --ignore=E501 ."
-script:
-  - bash run_tests.sh
-
-# Deploy to pypi automatically from tagged releases on the stable branch
-deploy:
-  provider: pypi
-  user: nikkiray
-  password:
-    secure: YN2r3txhbQI+hZUhUrKy4/dANHKTOz+fzLdHTwhjtPEjTUZvcmxhx582qc4rmPO1qvKH+bzjq2YhhO0J+uN7PmYAvMGPDu1Cjn46GiDogfq3C2+vkM+iovXmXXW+/pd5GRSD2I0+P7s3z1BG2iMwHXrynlxCa9mDApN4kJvEs98Z8SlpUpsOSvv/BhMTMaS1BXUR14ZDedvwK7YJmUbCfdnHufT1T8egRqxbwVyFXQujLpXCv1XDo0mNYjYMjh6DKkn/loT9ZAFSpNYFPdf/ljZIaWbNEqbJ//xXqStW4ix8dVgItN2sNJXPoEAKKptofqzGmmevph0FwBO0aeNmy+nV0tZHmzGk24ofJhjdYuwJTeKSYJBrK0Hye7sQV19G7rba9ZdMp8fO/pLEW6d6g20tABrLxJDtPM+dCL8Tqhy+G0XTY5lKC3x9o+RldGrJCdecL1g4G05DCNeA4YeEdn3/dKt9JjlSXIxwWAFGXhQQtpY3GBKknDIW5gvdxcIk/ktLg80M7IZ5vd/6urN63jGmffawiMJ2Fv+Gx4c1Twm9CA0H8yKH2fV5mepFplpYUPkFjCNP8P5P6VyePSVFa2Re4+UXgzncoupDhG/FDW+skvqRk3S+ga68cNSzDKi2WcdTpRLhS7bvb8yHzshZ2JBMko06mJtpoKfZ8tI8iZg=
-  on:
-    tags: true
-    distributions: sdist bdist_wheel
-    repo: airbnb/knowledge-repo
-    condition: $TRAVIS_PYTHON_VERSION = "3.4"
+sudo: required
+
+language: python
+python:
+  - "2.7"
+  - "3.4"
+
+services:
+  - docker
+
+env:
+  - PORT=80
+
+before_install:
+  # build docker image - this image contains none of the testing dependencies so is releasable if the tests pass
+  - docker-compose build
+install:
+  # create web app and system-under-test containers
+  - docker-compose up -d
+  # install python testing dependencies
+  - docker exec knowledgerepo_web_1 pip install autopep8 pep8 nose coverage bs4
+
+# Cache isn't working yet because problems with expiration header. Disable for now because it's breaking builds.
+# cache: pip
+
+before_script:
+  # make sure test .py files are nonexecutable so nose will find them
+  - docker exec knowledgerepo_web_1 bash -c 'chmod a-x /app/tests/*.py'
+  - "docker exec knowledgerepo_web_1 pep8 --exclude knowledge_repo/app/migrations,build,deploy,kube --ignore=E501 ."
+script:
+  # run unit tests
+  - docker exec knowledgerepo_web_1 bash run_tests.sh
+  # run system tests
+  - docker exec knowledgerepo_sut_1 bash run_tests.sh
+  # kill and remove the containers
+  - docker-compose down
+
+after_success:
+  # deploy successfully built images to docker hub
+  - docker login -u="$DOCKER_USERNAME" -p="$DOCKER_PASSWORD"
+  # omit docker image tags for successful builds on the master branch, pushing to :latest instead
+  - if [ "$TRAVIS_BRANCH" == "master" ];
+    then export DOCKER_REPO_FULLNAME=${DOCKER_REPO}-python${TRAVIS_PYTHON_VERSION};
+    else export DOCKER_REPO_FULLNAME=${DOCKER_REPO}-python${TRAVIS_PYTHON_VERSION}:${TRAVIS_BRANCH};
+    fi
+  - docker tag knowledgerepo_web $DOCKER_REPO_FULLNAME
+  - docker push $DOCKER_REPO_FULLNAME
+
+# Deploy to pypi automatically from tagged releases on the stable branch
+deploy:
+  provider: pypi
+  user: $PYPI_USER
+  password:
+    secure: $PYPI_PASS_SECURE
+  on:
+    tags: true
+    distributions: sdist bdist_wheel
+    repo: $PYPI_REPO
+    condition: $TRAVIS_PYTHON_VERSION = "3.4"

Dockerfile

@@ -0,0 +1,64 @@
+FROM ubuntu:16.04
+
+ARG TRAVIS_PYTHON_VERSION
+ARG PORT
+
+# Install required Ubuntu packages
+RUN apt-get update
+RUN apt-get install -y wget
+RUN apt-get install -y bzip2
+RUN apt-get install -y git
+
+# Set the locale
+RUN apt-get install -y locales
+RUN sed -i -e 's/# en_US.UTF-8 UTF-8/en_US.UTF-8 UTF-8/' /etc/locale.gen && \
+    locale-gen
+ENV LANG en_US.UTF-8
+ENV LANGUAGE en_US:en
+ENV LC_ALL en_US.UTF-8
+
+# Download appropriate version of Miniconda
+RUN if [[ "$TRAVIS_PYTHON_VERSION" == "2.7" ]]; then wget https://repo.continuum.io/miniconda/Miniconda2-latest-Linux-x86_64.sh -O miniconda.sh; \
+else wget https://repo.continuum.io/miniconda/Miniconda3-latest-Linux-x86_64.sh -O miniconda.sh; fi
+
+# Install Miniconda
+RUN bash miniconda.sh -b -p /miniconda
+ENV PATH=/miniconda/bin:${PATH}
+RUN hash -r
+
+# Set up conda package installer
+RUN conda config --set always_yes yes --set changeps1 no
+RUN conda update -q conda
+
+# Useful for debugging any issues with conda
+RUN conda info -a
+
+# Install R
+RUN conda install -c r r
+RUN conda install -c r r-knitr
+
+# Set the application directory
+WORKDIR /app
+
+# Install python dependencies - do this before adding rest of code to allow docker to cache this step
+ADD ./requirements.txt /app/requirements.txt
+RUN pip install -r requirements.txt
+
+# Copy our code from the current folder to /app inside the container
+ADD . /app
+
+# Run project installation scripts
+RUN python setup.py develop
+
+# Ready dependencies to use IpynbFormat instances
+RUN pip install --ignore-installed --upgrade nbformat nbconvert[execute] traitlets
+
+# Set up to use a new empty repo until configured otherwise
+RUN ./scripts/knowledge_repo --repo ./default_repo init
+ENV KNOWLEDGE_REPO=./default_repo
+
+EXPOSE ${PORT}
+ENV PORT=${PORT}
+
+# Deploy via gunicorn as standard startup command
+CMD ["bash", "-c", "./scripts/knowledge_repo deploy --port ${PORT}"]

Dockerfile.systest

@@ -0,0 +1,23 @@
+
+# this Dockerfile builds a container from which system tests can be run
+
+FROM ubuntu:16.04
+
+# Set the locale
+RUN apt-get update
+RUN apt-get install -y locales
+RUN sed -i -e 's/# en_US.UTF-8 UTF-8/en_US.UTF-8 UTF-8/' /etc/locale.gen && \
+    locale-gen
+ENV LANG en_US.UTF-8
+ENV LANGUAGE en_US:en
+ENV LC_ALL en_US.UTF-8
+
+ARG PORT
+
+RUN apt-get update && apt-get install -yq curl && apt-get clean
+
+WORKDIR /app
+
+ADD ./system_tests/run_tests.sh /app/run_tests.sh
+
+CMD ["bash"]