diff --git a/.gitignore b/.gitignore index b228f9c54..698e844a3 100644 --- a/.gitignore +++ b/.gitignore @@ -12,6 +12,7 @@ env/ # Django stuff: *.log *.sqlite3 +settings/dev.py # Sphinx documentation docs/build/ diff --git a/README.md b/README.md index d2ebbc228..1007676ac 100644 --- a/README.md +++ b/README.md @@ -10,7 +10,26 @@ Fabrik is an online collaborative platform to build, visualize and train deep le This app is presently under active development and we welcome contributions. Please check out our [issues thread](https://github.com/Cloud-CV/IDE/issues) to find things to work on, or ping us on [Gitter](https://gitter.im/Cloud-CV/IDE). -### How to setup + +## Installation Instructions + +Setting up Fabrik on your local machine is really easy. You can setup Fabrik using two methods: + +### Using Docker + +1. Get the source code on to your machine via git. + + ```shell + git clone https://github.com/Cloud-CV/Fabrik.git && cd Fabrik + ``` + +2. Build and run the Docker containers. This might take a while. You should be able to access Fabrik at `0.0.0.0:8000`. + + ``` + docker-compose up --build + ``` + +### Using Virtual Environment 1. First set up a virtualenv ``` sudo apt-get install python-pip python-dev python-virtualenv @@ -22,8 +41,13 @@ This app is presently under active development and we welcome contributions. Ple ``` git clone --recursive https://github.com/Cloud-CV/Fabrik.git ``` + +3. Rename settings/dev.sample.py as settings/dev.py and change credential in settings/dev.py + ``` + cp settings/dev.sample.py settings/dev.py + ``` -3. If you have Caffe, Keras and Tensorflow already installed on your computer, skip this step +4. If you have Caffe, Keras and Tensorflow already installed on your computer, skip this step * For Linux users ``` cd Fabrik/requirements @@ -42,7 +66,7 @@ This app is presently under active development and we welcome contributions. Ple * [Install Caffe](http://caffe.berkeleyvision.org/install_osx.html) * [Install Tensorflow](https://www.tensorflow.org/install/install_mac) * [Install Keras](https://keras.io/#installation) -4. Install dependencies +5. Install dependencies * For developers: ``` pip install -r requirements/dev.txt @@ -51,7 +75,7 @@ This app is presently under active development and we welcome contributions. Ple ``` pip install -r requirements/common.txt ``` -5. [Install postgres >= 9.5](https://www.digitalocean.com/community/tutorials/how-to-install-and-use-postgresql-on-ubuntu-16-04) +6. [Install postgres >= 9.5](https://www.digitalocean.com/community/tutorials/how-to-install-and-use-postgresql-on-ubuntu-16-04) * Setup postgres database * Start postgresql by typing ```sudo service postgresql start``` * Now login as user postgres by running ```sudo -u postgres psql``` and type the commands below @@ -71,7 +95,7 @@ This app is presently under active development and we welcome contributions. Ple python manage.py makemigrations caffe_app python manage.py migrate ``` -6. Install node modules +7. Install node modules ``` npm install sudo npm install -g webpack diff --git a/docker-compose.yml b/docker-compose.yml new file mode 100644 index 000000000..ec828db2e --- /dev/null +++ b/docker-compose.yml @@ -0,0 +1,49 @@ +version: '2' + +services: + db: + hostname: db + image: postgres + ports: + - "5432:5432" + networks: + - fabrik + + reactjs: + container_name: reactjs + hostname: reactjs + build: + context: ./ + dockerfile: docker/dev/reactjs/Dockerfile + environment: + NODE_ENV: development + volumes: + - .:/code + - /code/node_modules + networks: + - fabrik + + django: + restart: always + container_name: django + hostname: django + env_file: + - docker/dev.env + links: + - reactjs + build: + context: ./ + dockerfile: docker/dev/django/Dockerfile + ports: + - "8000:8000" + command: ["./docker/wait-for-it.sh", "db:5432", "--", "sh", "/code/docker/dev/django/container-start.sh"] + depends_on: + - db + - reactjs + volumes: + - .:/code + networks: + - fabrik + +networks: + fabrik: diff --git a/docker/dev.env b/docker/dev.env new file mode 100644 index 000000000..eed96e454 --- /dev/null +++ b/docker/dev.env @@ -0,0 +1,9 @@ +DEBUG=True + +POSTGRES_NAME=postgres +POSTGRES_USER=postgres +POSTGRES_PASSWORD=postgres +POSTGRES_HOST=db +POSTGRES_PORT=5432 + +DJANGO_SERVER=django diff --git a/docker/dev/django/Dockerfile b/docker/dev/django/Dockerfile new file mode 100644 index 000000000..d1b595299 --- /dev/null +++ b/docker/dev/django/Dockerfile @@ -0,0 +1,54 @@ +FROM ubuntu:16.04 +MAINTAINER CloudCV Team + +RUN apt-get update && apt-get install -y --no-install-recommends \ + build-essential \ + cmake \ + git \ + wget \ + libatlas-base-dev \ + libboost-all-dev \ + libgflags-dev \ + libgoogle-glog-dev \ + libhdf5-serial-dev \ + libleveldb-dev \ + liblmdb-dev \ + libopencv-dev \ + libprotobuf-dev \ + libsnappy-dev \ + protobuf-compiler \ + python-dev \ + python-numpy \ + python-pip \ + python-setuptools \ + python-scipy && \ + rm -rf /var/lib/apt/lists/* + +ENV CAFFE_ROOT=$HOME/caffe + +WORKDIR $CAFFE_ROOT + +ENV CLONE_TAG=1.0 + +RUN pip install --upgrade pip + +RUN git clone -b ${CLONE_TAG} --depth 1 https://github.com/BVLC/caffe.git . && \ + cd python && for req in $(cat requirements.txt) pydot; do pip install $req; done && cd .. && \ + mkdir build && cd build && \ + cmake -DCPU_ONLY=1 .. && \ + make -j"$(nproc)" + +ENV PYCAFFE_ROOT $CAFFE_ROOT/python +ENV PYTHONPATH $PYCAFFE_ROOT:$PYTHONPATH +ENV PATH $CAFFE_ROOT/build/tools:$PYCAFFE_ROOT:$PATH +RUN echo "$CAFFE_ROOT/build/lib" >> /etc/ld.so.conf.d/caffe.conf && ldconfig + +WORKDIR /code + +ADD . /code + +COPY settings/dev.sample.py settings/dev.py + +RUN pip install -r requirements/dev.txt + +EXPOSE 8000 diff --git a/docker/dev/django/container-start.sh b/docker/dev/django/container-start.sh new file mode 100755 index 000000000..88e356076 --- /dev/null +++ b/docker/dev/django/container-start.sh @@ -0,0 +1,4 @@ +#!/bin/sh +cd /code && \ +python manage.py migrate --noinput && \ +KERAS_BACKEND=theano python manage.py runserver 0.0.0.0:8000 diff --git a/docker/dev/reactjs/.dockerignore b/docker/dev/reactjs/.dockerignore new file mode 100644 index 000000000..93f136199 --- /dev/null +++ b/docker/dev/reactjs/.dockerignore @@ -0,0 +1,2 @@ +node_modules +npm-debug.log diff --git a/docker/dev/reactjs/Dockerfile b/docker/dev/reactjs/Dockerfile new file mode 100644 index 000000000..27eeb8ce5 --- /dev/null +++ b/docker/dev/reactjs/Dockerfile @@ -0,0 +1,29 @@ +FROM ubuntu:16.04 + +MAINTAINER CloudCV Team + +RUN apt-get update -qq && apt-get install -y build-essential git curl libfontconfig + +RUN apt-get install nodejs-legacy -y + +RUN apt-get install npm -y + +RUN apt-get install -y ruby-dev + +RUN gem install sass -v 3.2.19 + +WORKDIR /code + +ADD . /code + +RUN npm link gulp + +RUN npm cache clean -f +RUN npm install -g n +RUN n stable + +RUN npm install + +RUN npm install -g webpack@1.13.1 + +CMD ["webpack", "--progress", "--watch", "--colors"] diff --git a/docker/wait-for-it.sh b/docker/wait-for-it.sh new file mode 100755 index 000000000..eca6c3b9c --- /dev/null +++ b/docker/wait-for-it.sh @@ -0,0 +1,161 @@ +#!/usr/bin/env bash +# Use this script to test if a given TCP host/port are available + +cmdname=$(basename $0) + +echoerr() { if [[ $QUIET -ne 1 ]]; then echo "$@" 1>&2; fi } + +usage() +{ + cat << USAGE >&2 +Usage: + $cmdname host:port [-s] [-t timeout] [-- command args] + -h HOST | --host=HOST Host or IP under test + -p PORT | --port=PORT TCP port under test + Alternatively, you specify the host and port as host:port + -s | --strict Only execute subcommand if the test succeeds + -q | --quiet Don't output any status messages + -t TIMEOUT | --timeout=TIMEOUT + Timeout in seconds, zero for no timeout + -- COMMAND ARGS Execute command with args after the test finishes +USAGE + exit 1 +} + +wait_for() +{ + if [[ $TIMEOUT -gt 0 ]]; then + echoerr "$cmdname: waiting $TIMEOUT seconds for $HOST:$PORT" + else + echoerr "$cmdname: waiting for $HOST:$PORT without a timeout" + fi + start_ts=$(date +%s) + while : + do + (echo > /dev/tcp/$HOST/$PORT) >/dev/null 2>&1 + result=$? + if [[ $result -eq 0 ]]; then + end_ts=$(date +%s) + echoerr "$cmdname: $HOST:$PORT is available after $((end_ts - start_ts)) seconds" + break + fi + sleep 1 + done + return $result +} + +wait_for_wrapper() +{ + # In order to support SIGINT during timeout: http://unix.stackexchange.com/a/57692 + if [[ $QUIET -eq 1 ]]; then + timeout $TIMEOUT $0 --quiet --child --host=$HOST --port=$PORT --timeout=$TIMEOUT & + else + timeout $TIMEOUT $0 --child --host=$HOST --port=$PORT --timeout=$TIMEOUT & + fi + PID=$! + trap "kill -INT -$PID" INT + wait $PID + RESULT=$? + if [[ $RESULT -ne 0 ]]; then + echoerr "$cmdname: timeout occurred after waiting $TIMEOUT seconds for $HOST:$PORT" + fi + return $RESULT +} + +# process arguments +while [[ $# -gt 0 ]] +do + case "$1" in + *:* ) + hostport=(${1//:/ }) + HOST=${hostport[0]} + PORT=${hostport[1]} + shift 1 + ;; + --child) + CHILD=1 + shift 1 + ;; + -q | --quiet) + QUIET=1 + shift 1 + ;; + -s | --strict) + STRICT=1 + shift 1 + ;; + -h) + HOST="$2" + if [[ $HOST == "" ]]; then break; fi + shift 2 + ;; + --host=*) + HOST="${1#*=}" + shift 1 + ;; + -p) + PORT="$2" + if [[ $PORT == "" ]]; then break; fi + shift 2 + ;; + --port=*) + PORT="${1#*=}" + shift 1 + ;; + -t) + TIMEOUT="$2" + if [[ $TIMEOUT == "" ]]; then break; fi + shift 2 + ;; + --timeout=*) + TIMEOUT="${1#*=}" + shift 1 + ;; + --) + shift + CLI="$@" + break + ;; + --help) + usage + ;; + *) + echoerr "Unknown argument: $1" + usage + ;; + esac +done + +if [[ "$HOST" == "" || "$PORT" == "" ]]; then + echoerr "Error: you need to provide a host and port to test." + usage +fi + +TIMEOUT=${TIMEOUT:-15} +STRICT=${STRICT:-0} +CHILD=${CHILD:-0} +QUIET=${QUIET:-0} + +if [[ $CHILD -gt 0 ]]; then + wait_for + RESULT=$? + exit $RESULT +else + if [[ $TIMEOUT -gt 0 ]]; then + wait_for_wrapper + RESULT=$? + else + wait_for + RESULT=$? + fi +fi + +if [[ $CLI != "" ]]; then + if [[ $RESULT -ne 0 && $STRICT -eq 1 ]]; then + echoerr "$cmdname: strict mode, refusing to execute subprocess" + exit $RESULT + fi + exec $CLI +else + exit $RESULT +fi diff --git a/ide/wsgi.py b/ide/wsgi.py index 4df30b0f3..4a21d43a2 100644 --- a/ide/wsgi.py +++ b/ide/wsgi.py @@ -11,6 +11,6 @@ from django.core.wsgi import get_wsgi_application -os.environ.setdefault("DJANGO_SETTINGS_MODULE", "ide.settings") +os.environ.setdefault("DJANGO_SETTINGS_MODULE", "settings") application = get_wsgi_application() diff --git a/manage.py b/manage.py index a3bf0edbe..f9726f9e6 100755 --- a/manage.py +++ b/manage.py @@ -3,7 +3,7 @@ import sys if __name__ == "__main__": - os.environ.setdefault("DJANGO_SETTINGS_MODULE", "ide.settings") + os.environ.setdefault("DJANGO_SETTINGS_MODULE", "settings") from django.core.management import execute_from_command_line diff --git a/requirements/caffe_tensorflow_keras_install.sh b/requirements/caffe_tensorflow_keras_install.sh index eaaf9b84b..b716966f8 100644 --- a/requirements/caffe_tensorflow_keras_install.sh +++ b/requirements/caffe_tensorflow_keras_install.sh @@ -1,12 +1,12 @@ # Generic dependencies echo "Installing generic dependencies" -sudo apt-get install git libatlas-base-dev python-protobuf python-numpy python-scipy python-h5py unzip make libblas-dev liblapack-dev libatlas-base-dev gfortran python-pip python-dev +apt-get install git libatlas-base-dev python-protobuf python-numpy python-scipy python-h5py unzip make libblas-dev liblapack-dev libatlas-base-dev gfortran python-pip python-dev pip install numpy scipy scikit-image #Caffe specific dependencies echo "Installing caffe specific dependencies" -sudo apt-get install libprotobuf-dev libleveldb-dev libsnappy-dev libopencv-dev libhdf5-serial-dev libgflags-dev libgoogle-glog-dev liblmdb-dev protobuf-compiler cmake -sudo apt-get install --no-install-recommends libboost-all-dev +apt-get install libprotobuf-dev libleveldb-dev libsnappy-dev libopencv-dev libhdf5-serial-dev libgflags-dev libgoogle-glog-dev liblmdb-dev protobuf-compiler cmake +apt-get install --no-install-recommends libboost-all-dev if [ ! -d $HOME/caffe/caffe ]; then #Download caffe diff --git a/requirements/common.txt b/requirements/common.txt index 4a71e3359..69427eac8 100644 --- a/requirements/common.txt +++ b/requirements/common.txt @@ -1,4 +1,7 @@ -Django==1.9.3 +django==1.10.2 pyaml==15.8.2 django-allauth==0.30.0 psycopg2==2.7.3 +tensorflow==1.4.1 +theano==0.9.0 +keras==2.0.8 diff --git a/settings/__init__.py b/settings/__init__.py new file mode 100644 index 000000000..67398ad77 --- /dev/null +++ b/settings/__init__.py @@ -0,0 +1,10 @@ +# -*- coding: utf-8 -*- +from __future__ import absolute_import + +# TODO: Add support for test and production environment settings + +try: + from .dev import * + print("Using Dev settings") +except ImportError: + pass diff --git a/ide/settings.py b/settings/common.py similarity index 60% rename from ide/settings.py rename to settings/common.py index 497b0f284..53f41ae57 100644 --- a/ide/settings.py +++ b/settings/common.py @@ -1,30 +1,12 @@ -""" -Django settings for ide project. - -Generated by 'django-admin startproject' using Django 1.9.3. - -For more information on this file, see -https://docs.djangoproject.com/en/1.9/topics/settings/ - -For the full list of settings and their values, see -https://docs.djangoproject.com/en/1.9/ref/settings/ -""" - import os # Build paths inside the project like this: os.path.join(BASE_DIR, ...) BASE_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) STATICFILES_DIRS = ( - os.path.normpath(os.path.join(os.path.abspath(os.path.dirname(__file__)), 'static')), - # Put strings here, like "/home/html/static" or "C:/www/django/static". - # Always use forward slashes, even on Windows. - # Don't forget to use absolute paths, not relative paths. + os.path.join(BASE_DIR, 'ide/static'), ) -# Quick-start development settings - unsuitable for production -# See https://docs.djangoproject.com/en/1.9/howto/deployment/checklist/ - # SECURITY WARNING: keep the secret key used in production secret! SECRET_KEY = 's9&vp1jq1yzr!1c_temg#v_)j-a)i5+@vbsekmi6pbjl4l1&u@' @@ -85,41 +67,6 @@ SITE_ID = 1 - -# Database -# https://docs.djangoproject.com/en/1.9/ref/settings/#databases - -DATABASES = { - 'default': { - 'ENGINE': 'django.db.backends.postgresql_psycopg2', - 'NAME': 'fabrik', - 'USER': 'admin', - 'PASSWORD': 'fabrik', - 'HOST': 'localhost', - 'PORT': '', - } -} - - -# Password validation -# https://docs.djangoproject.com/en/1.9/ref/settings/#auth-password-validators - -AUTH_PASSWORD_VALIDATORS = [ - { - 'NAME': 'django.contrib.auth.password_validation.UserAttributeSimilarityValidator', - }, - { - 'NAME': 'django.contrib.auth.password_validation.MinimumLengthValidator', - }, - { - 'NAME': 'django.contrib.auth.password_validation.CommonPasswordValidator', - }, - { - 'NAME': 'django.contrib.auth.password_validation.NumericPasswordValidator', - }, -] - - # Internationalization # https://docs.djangoproject.com/en/1.9/topics/i18n/ diff --git a/settings/dev.sample.py b/settings/dev.sample.py new file mode 100644 index 000000000..89ed79aab --- /dev/null +++ b/settings/dev.sample.py @@ -0,0 +1,18 @@ +import os +from .common import * + +# Database +# https://docs.djangoproject.com/en/1.9/ref/settings/#databases + +DEBUG = True + +DATABASES = { + 'default': { + 'ENGINE': 'django.db.backends.postgresql_psycopg2', + 'NAME': os.environ.get("POSTGRES_NAME", 'postgres'), + 'USER': os.environ.get("POSTGRES_USER", 'postgres'), + 'PASSWORD': os.environ.get("POSTGRES_PASSWORD", 'postgres'), + 'HOST': os.environ.get("POSTGRES_HOST", 'db'), + 'PORT': os.environ.get("POSTGRES_PORT", 5432), + } +}