Dev build and run process (ansible#107)

* Adds development container build

* Finish dev build process. Remove unneeded bits.

* Always console log. Don't require an email address.

* First go at contributing

Makefile

@@ -1,8 +1,6 @@
 PYTHON=python
 SITELIB=$(shell $(PYTHON) -c "from distutils.sysconfig import get_python_lib; print get_python_lib()")
 
-# Get the branch information from git
-GIT_DATE := $(shell git log -n 1 --format="%ai")
 DATE := $(shell date -u +%Y%m%d%H%M)
 
 VERSION=$(shell $(PYTHON) -c "from galaxy import __version__; print(__version__.split('-')[0])")
@@ -28,83 +26,89 @@ DEB_BUILD_DIR=deb-build/galaxy-$(VERSION)
 DEB_PKG_RELEASE=$(VERSION)-$(RELEASE)
 endif
 
-.PHONY: clean clean_dist clean_images clean_containers \
-        refresh migrate migrate_empty makemigrations build_from_scratch \
-        build build_debug \
-        run run_production \
-        flake8 test waitenv \
-        build_indexes \
-        sdist stop stop_production requirements ui_build export_test_data import_test_data createsuperuser \
-        refresh_role_counts shell
+VENV_BIN=/var/lib/galaxy/venv/bin
 
-# Remove containers, images and ~/.galaxy
-clean:
-	@./clean.sh
+.PHONY: clean_dist \
+        sdist requirements ui_build export_test_data import_test_data \
+        refresh_role_counts
 
-# Refresh development environment after pulling new code.
-refresh: clean build run
+.PHONY: dev/build
+dev/build:
+	scripts/build-docker-dev.sh
 
-# Remove containers
-clean_containers:
-	./clean.sh containers
-
-# Remove images
-clean_images:
-	./clean.sh images
-
-# Create and execute database migrations
-migrate:
-	@docker exec -i -t galaxy_django_1 /venv/bin/python ./manage.py makemigrations main --noinput
-	@docker exec -i -t galaxy_django_1 /venv/bin/python ./manage.py migrate --noinput
-
-# Create an empty migration
-migrate_empty:
-	@docker exec -i -t galaxy_django_1 /venv/bin/python ./manage.py makemigrations --empty main
-
-makemigrations:
-	@docker exec -i -t galaxy_django_1 /venv/bin/python ./manage.py makemigrations main
-
-psql:
-	@docker exec -i -t galaxy_django_1 psql -h postgres -d galaxy -U galaxy
-
-# Build Galaxy images
-build:
-	ansible-container --debug --devel build
-
-# Rebuild Galaxy search indexes
-build_indexes:
-	@echo "Rebuild Custom Indexes"
-	@docker exec -i -t galaxy_django_1 /venv/bin/python ./manage.py rebuild_galaxy_indexes
-	@echo "Rebuild Search Index"
-	@docker exec -i -t galaxy_django_1 /venv/bin/python ./manage.py rebuild_index --noinput
-
-createsuperuser:
+.PHONY: dev/createsuperuser
+dev/createsuperuser:
 	@echo "Create Superuser"
-	@docker exec -i -t galaxy_django_1 /venv/bin/python ./manage.py  createsuperuser
-
-# Run flake8 inside the django container
-flake8:
-	docker run -v ${PWD}:/galaxy -w /galaxy galaxy-django:latest /venv/bin/flake8 --config=tox.ini galaxy
-
-test:
-	docker exec -i -t galaxy_django_1 /venv/bin/python ./manage.py test
-
-waitenv:
-	docker exec -i -t galaxy_django_1 /venv/bin/python ./manage.py waitenv
-
-# Start Galaxy containers with django and gulp in the foreground
-run:
-	ansible-container --debug --devel run
-
-# Start production
-run_production:
-	ansible-container --debug --devel run --production
-
-stop:
-	@ansible-container stop --force
-
-stop_production:
-	@ansible-container stop --force --production
+	@docker-compose -f ./scripts/compose-dev.yml -p galaxy exec galaxy bash -c '${VENV_BIN}/python ./manage.py createsuperuser'
+
+.PHONY: dev/migrate
+dev/migrate:
+	docker-compose -f ./scripts/compose-dev.yml -p galaxy exec galaxy bash -c '${VENV_BIN}/python ./manage.py migrate --noinput'
+
+.PHONY: dev/makemigrations
+dev/makemigrations:
+	docker-compose -f ./scripts/compose-dev.yml -p galaxy exec galaxy bash -c '${VENV_BIN}/python ./manage.py makemigrations main'
+
+.PHONY: dev/flake8
+dev/flake8:
+	@echo "Running flake8"
+	@docker-compose -f ./scripts/compose-dev.yml -p galaxy exec galaxy bash -c '${VENV_BIN}/flake8 --config=tox.ini galaxy'
+
+.PHONY: dev/test
+dev/test:
+	@echo "Running tests"
+	@docker-compose -f ./scripts/compose-dev.yml -p galaxy exec galaxy bash -c '${VENV_BIN}/python ./manage.py test'
+
+.PHONY: dev/up
+dev/up:
+	docker-compose -f ./scripts/compose-dev.yml -p galaxy up
+
+.PHONY: dev/up_detached
+dev/up_detached:
+	# Start all containers detached
+	docker-compose -f ./scripts/compose-dev.yml -p galaxy up -d
+
+.PHONY: dev/up_tmux
+dev/up_tmux:
+	# Run before dev/tmux to start containers detached and no processes running in the galaxy container.
+	TMUX=1 docker-compose -f ./scripts/compose-dev.yml -p galaxy up -d
+
+.PHONY: dev/down
+dev/down:
+	docker-compose -f ./scripts/compose-dev.yml -p galaxy down
+
+.PHONY: dev/restart
+dev/restart:
+	# Restart one or more services
+	docker-compose -f ./scripts/compose-dev.yml -p galaxy restart $(filter-out $@,$(MAKECMDGOALS)) 
+
+.PHONY: dev/stop
+dev/stop:
+	# Stop one or more services
+	docker-compose -f ./scripts/compose-dev.yml -p galaxy stop $(filter-out $@,$(MAKECMDGOALS)) 
+
+.PHONY: dev/tmux_noattach
+dev/tmux_noattach:
+	# Create the tmux session. Do NOT call directly. Use dev/tmux or dev/tmuxcc instead.
+	tmux new-session -d -s galaxy -n galaxy 'bash -c "make runserver; exec bash"' 
+	tmux new-window -t galaxy:1 -n celery 'bash -c "make celery; exec bash"'
+	tmux new-window -t galaxy:2 -n gulp 'bash -c "make gulp; exec bash"'
+	tmux select-window -t galaxy:0
+
+.PHONY: dev/tmux
+dev/tmux: 
+	# Connect to the galaxy container, start processes, and pipe stdout/stderr through a tmux session
+	docker-compose -f ./scripts/compose-dev.yml -p galaxy exec galaxy bash -c 'make dev/tmux_noattach; tmux -2 attach-session -t galaxy'
+
+.PHONY: dev/tmuxcc
+dev/tmuxcc: dev/tmux_noattach
+	# Same as above using iTerm's built-in tmux support
+	docker-compose -f ./scripts/compose-dev.yml -p galaxy exec galaxy bash -c 'make dev/tmux_noattach; tmux -2 -CC attach-session -t galaxy'
+
+.PHONY: dev/gulp_build
+dev/gulp_build:
+	# build UI components
+	docker-compose -f ./scripts/compose-dev.yml -p galaxy exec galaxy bash -c '/usr/local/bin/gulp build'
 
 sdist: clean_dist ui_build
 	if [ "$(OFFICIAL)" = "yes" ] ; then \
@@ -113,25 +117,13 @@ sdist: clean_dist ui_build
 	   BUILD=$(BUILD) $(PYTHON) setup.py sdist_galaxy; \
 	fi
 
-requirements:
-	@if [ "$(VIRTUAL_ENV)" ]; then \
-	    pip install distribute==0.7.3; \
-	    pip install -r requirements.txt; \
-	    $(PYTHON) fix_virtualenv_setuptools.py; \
-	else \
-	    sudo pip install -r requirements.txt; \
-	fi
-
 clean_dist:
 	rm -rf dist/*
 	rm -rf build rpm-build *.egg-info
 	rm -rf debian deb-build
 	rm -f galaxy/static/dist/*.js
 	find . -type f -regex ".*\.py[co]$$" -delete
 
-ui_build:
-	node node_modules/gulp/bin/gulp.js build
-
 export_test_data:
 	@echo Export data to test-data/role_data.dmp.gz
 	@docker exec -i -t galaxy_django_1 /galaxy/test-data/export.sh
@@ -144,7 +136,34 @@ refresh_role_counts:
 	@echo Refresh role counts
 	@docker exec -i -t galaxy_django_1 /venv/bin/python ./manage.py refresh_role_counts
 
-shell:
-	@echo Starting shell on galaxy_django_1
-	@docker exec -i -t galaxy_django_1 /bin/bash
+.PHONY: celery
+celery:
+	${VENV_DIR}/bin/python manage.py celeryd -B --autoreload -Q 'celery,import_tasks,login_tasks'
+
+.PHONY: runserver
+runserver:
+	${VENV_DIR}/bin/python manage.py runserver 0.0.0.0:8888
+
+.PHONY: gulp
+gulp:
+	/usr/local/bin/gulp
+
+.PHONY: waitenv
+waitenv:
+	@echo "Waiting for services to start..."
+	${VENV_DIR}/bin/python ./manage.py waitenv
+
+.PHONY: migrate
+migrate:
+	@echo "Run migrations"
+	${VENV_DIR}/bin/python ./manage.py migrate --noinput
+
+.PHONY: build_indexes
+build_indexes:
+	@echo "Rebuild Custom Indexes"
+	${VENV_DIR}/bin/python ./manage.py rebuild_galaxy_indexes
+	@echo "Rebuild Search Index"
+	${VENV_DIR}/bin/python ./manage.py rebuild_index --noinput
 
+%:      
+	@:

galaxy/settings/default.py

@@ -266,7 +266,7 @@
 
 ACCOUNT_EMAIL_REQUIRED = True
 
-ACCOUNT_EMAIL_VERIFICATION = 'mandatory'
+ACCOUNT_EMAIL_VERIFICATION = 'none'
 
 ACCOUNT_EMAIL_SUBJECT_PREFIX = 'Ansible Galaxy '
 
@@ -326,7 +326,7 @@
 # Logging
 # =========================================================
 
-LOGS_DIR = os.path.join(BASE_DIR, 'var', 'log')
+# LOGS_DIR = os.path.join(BASE_DIR, 'var', 'log')
 
 # TODO(cutwater): Adjust logging config for production environment
 # TODO(cutwater): Review logging config
@@ -356,30 +356,30 @@
             'filters': ['require_debug_false'],
             'class': 'django.utils.log.AdminEmailHandler'
         },
-        'allauth_logfile': {
-            'level': 'DEBUG',
-            'class': 'logging.handlers.RotatingFileHandler',
-            'filename': os.path.join(LOGS_DIR, 'allauth.log'),
-            'maxBytes': 500000,
-            'backupCount': 4,
-            'formatter': 'verbose',
-        },
-        'django_logfile': {
-            'level': 'DEBUG',
-            'class': 'logging.handlers.RotatingFileHandler',
-            'filename': os.path.join(LOGS_DIR, 'django.log'),
-            'maxBytes': 500000,
-            'backupCount': 4,
-            'formatter': 'verbose',
-        },
-        'galaxy_logfile': {
-            'level': 'DEBUG',
-            'class': 'logging.handlers.RotatingFileHandler',
-            'filename': os.path.join(LOGS_DIR, 'galaxy.log'),
-            'maxBytes': 500000,
-            'backupCount': 4,
-            'formatter': 'verbose',
-        },
+#        'allauth_logfile': {
+#            'level': 'DEBUG',
+#            'class': 'logging.handlers.RotatingFileHandler',
+#            'filename': os.path.join(LOGS_DIR, 'allauth.log'),
+#            'maxBytes': 500000,
+#            'backupCount': 4,
+#            'formatter': 'verbose',
+#        },
+#        'django_logfile': {
+#            'level': 'DEBUG',
+#            'class': 'logging.handlers.RotatingFileHandler',
+#            'filename': os.path.join(LOGS_DIR, 'django.log'),
+#            'maxBytes': 500000,
+#            'backupCount': 4,
+#            'formatter': 'verbose',
+#        },
+#        'galaxy_logfile': {
+#            'level': 'DEBUG',
+#            'class': 'logging.handlers.RotatingFileHandler',
+#            'filename': os.path.join(LOGS_DIR, 'galaxy.log'),
+#            'maxBytes': 500000,
+#            'backupCount': 4,
+#            'formatter': 'verbose',
+#        },
         'console': {
             'level': 'INFO',
             'class': 'logging.StreamHandler',
@@ -394,7 +394,7 @@
             'propagate': True,
         },
         'django': {
-            'handlers': ['django_logfile'],
+            'handlers': ['console'],
             'level': 'DEBUG',
             'propagate': True,
         },

galaxy/settings/development.py

@@ -110,7 +110,7 @@
 # Celery settings
 # ---------------------------------------------------------
 
-BROKER_URL = 'amqp://galaxy:galaxy@rabbit:5672/galaxy'
+BROKER_URL = 'amqp://galaxy:galaxy@rabbitmq:5672/galaxy'
 
 # =========================================================
 # Galaxy Settings
@@ -122,7 +122,7 @@
 
 WAIT_FOR = [
     {'host': 'postgres', 'port': 5432},
-    {'host': 'rabbit', 'port': 5672},
+    {'host': 'rabbitmq', 'port': 5672},
     {'host': 'memcache', 'port': 11211},
     {'host': 'elastic', 'port': 9200}
 ]

gulpfile.js

@@ -12,7 +12,7 @@ const history  = require('connect-history-api-fallback')
 
 
 gulp.task('server', function() {
-    const proxyDjango = proxy('/', {target: 'http://django:8000', xfwd: true})
+    const proxyDjango = proxy('/', {target: 'http://localhost:8888', xfwd: true})
 
     sync.init({
         notify: false,

scripts/build-docker-dev.sh

@@ -0,0 +1,9 @@
+#!/bin/bash
+
+set -o nounset
+set -o errexit
+
+docker build --rm -t galaxy-build -f scripts/docker-release/Dockerfile.build .
+
+docker build --rm -t galaxy-dev -f scripts/docker-dev/Dockerfile .
+

scripts/compose-dev.yml

@@ -0,0 +1,37 @@
+version: '3'
+
+services:
+  galaxy:
+    image: galaxy-dev:latest
+    build:
+      context: ./docker-dev
+    volumes:
+      - ${PWD}:/galaxy:rw
+    environment:
+      - TMUX
+    ports:
+      - '8000:8000'
+    links:
+      - postgres
+      - elastic
+      - memcache
+      - rabbitmq
+  postgres:
+    image: postgres:9.5.4
+    environment:
+      - POSTGRES_DB=galaxy
+      - POSTGRES_USER=galaxy
+      - POSTGRES_PASSWORD=galaxy
+
+  elastic:
+    image: elasticsearch:2.4.1
+
+  memcache:
+    image: memcached:latest
+
+  rabbitmq:
+    image: rabbitmq:latest
+    environment:
+      - RABBITMQ_DEFAULT_USER=galaxy
+      - RABBITMQ_DEFAULT_PASS=galaxy
+      - RABBITMQ_DEFAULT_VHOST=galaxy