From 449628ed73d05fe60a4da4e2a387a5764c539329 Mon Sep 17 00:00:00 2001 From: Chris Houseknecht Date: Fri, 27 Oct 2017 09:41:44 +0200 Subject: [PATCH] Dev build and run process (#107) * Adds development container build * Finish dev build process. Remove unneeded bits. * Always console log. Don't require an email address. * First go at contributing --- CONTRIBUTING.md | 281 +++++++++++------- Makefile | 199 +++++++------ galaxy/settings/default.py | 54 ++-- galaxy/settings/development.py | 4 +- gulpfile.js | 2 +- container.yml => old-scripts/container.yml | 0 develop.yml => old-scripts/develop.yml | 0 .../fix_virtualenv_setuptools.py | 0 {roles => old-scripts/roles}/README.md | 0 .../roles}/django-role/defaults/main.yml | 0 .../roles}/django-role/files/common.sh | 0 .../roles}/django-role/files/inventory | 0 .../django-role/files/start-development.sh | 0 .../roles}/django-role/files/start.sh | 0 .../roles}/django-role/files/update_site.sql | 0 .../tasks/install-python-packages.yml | 0 .../roles}/django-role/tasks/main.yml | 0 .../roles}/django-role/tasks/setup-galaxy.yml | 0 .../roles}/django-role/templates/pgpass.j2 | 0 .../roles}/gulp-role/tasks/main.yml | 0 {roles => old-scripts/roles}/nginx/AUTHORS | 0 .../roles}/nginx/defaults/main.yml | 0 .../roles}/nginx/files/nginx.repo | 0 .../roles}/nginx/tasks/main.yml | 0 .../roles}/nginx/templates/nginx.conf.j2 | 0 scripts/build-docker-dev.sh | 9 + scripts/compose-dev.yml | 37 +++ scripts/docker-dev/Dockerfile | 18 ++ scripts/docker-dev/Procfile | 3 + scripts/docker-dev/sleep.sh | 6 + scripts/docker-dev/start-develop.sh | 16 + scripts/docker-release/Dockerfile.build | 3 +- var/email/.gitkeep | 0 var/log/.gitkeep | 0 34 files changed, 407 insertions(+), 225 deletions(-) rename container.yml => old-scripts/container.yml (100%) rename develop.yml => old-scripts/develop.yml (100%) rename fix_virtualenv_setuptools.py => old-scripts/fix_virtualenv_setuptools.py (100%) rename {roles => old-scripts/roles}/README.md (100%) rename {roles => old-scripts/roles}/django-role/defaults/main.yml (100%) rename {roles => old-scripts/roles}/django-role/files/common.sh (100%) rename {roles => old-scripts/roles}/django-role/files/inventory (100%) rename {roles => old-scripts/roles}/django-role/files/start-development.sh (100%) rename {roles => old-scripts/roles}/django-role/files/start.sh (100%) rename {roles => old-scripts/roles}/django-role/files/update_site.sql (100%) rename {roles => old-scripts/roles}/django-role/tasks/install-python-packages.yml (100%) rename {roles => old-scripts/roles}/django-role/tasks/main.yml (100%) rename {roles => old-scripts/roles}/django-role/tasks/setup-galaxy.yml (100%) rename {roles => old-scripts/roles}/django-role/templates/pgpass.j2 (100%) rename {roles => old-scripts/roles}/gulp-role/tasks/main.yml (100%) rename {roles => old-scripts/roles}/nginx/AUTHORS (100%) rename {roles => old-scripts/roles}/nginx/defaults/main.yml (100%) rename {roles => old-scripts/roles}/nginx/files/nginx.repo (100%) rename {roles => old-scripts/roles}/nginx/tasks/main.yml (100%) rename {roles => old-scripts/roles}/nginx/templates/nginx.conf.j2 (100%) create mode 100755 scripts/build-docker-dev.sh create mode 100644 scripts/compose-dev.yml create mode 100644 scripts/docker-dev/Dockerfile create mode 100644 scripts/docker-dev/Procfile create mode 100755 scripts/docker-dev/sleep.sh create mode 100755 scripts/docker-dev/start-develop.sh delete mode 100644 var/email/.gitkeep delete mode 100644 var/log/.gitkeep diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 6beb341a1..16245d747 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -1,91 +1,215 @@ # Contributing -## Development +Hi! We're excited to have you as a contributor. -To setup a local development environment you will need to install the following: +Have questions about this document or anything not covered here? Come chat with us at `#ansible-community` on irc.freenode.net. -* [Ansible Container](https://github.com/ansible/ansible-container) running from source. For assistance, see [the Running from Source guide](http://docs.ansible.com/ansible-container/installation.html#running-from-source). +## Things to know prior to submitting code -* We recommend using [Git Flow](https://github.com/nvie/gitflow), although it's not strictly required. Any development -should be done in feature branches, and compared to the `develop` branch. +- All code submissions are done through pull requests against the `develop` branch. +- You must use `git commit --signoff` for any commit to be merged, and agree that usage of --signoff constitutes agreement with the terms of [DCO 1.1](./DCO_1_1.md). +- Take care to make sure no merge commits are in the submission, and use `git rebase` vs `git merge` for this reason. +- If submitting a large code change, it's a good idea to join the `#ansible-community` channel on irc.freenode.net, and talk about what you would like to do or add first. This not only helps everyone know what's going on, it also helps save time and effort, if the community decides some changes are needed. +- We ask all of our community members and contributors to adhere to the [Ansible code of conduct](http://docs.ansible.com/ansible/latest/community/code_of_conduct.html). If you have questions, or need assistance, please reach out to our community team at [codeofconduct@ansible.com](mailto:codeofconduct@ansible.com) -### Start a Feature +## Setting up your development environment -Clone the [Galaxy repo](https://github.com/ansible/galaxy) to your local projects folder: +The Galaxy development environment workflow and toolchain is based on Docker, and the docker-compose tool, to provide dependencies, services, and databases necessary to run all of the components. It also binds the local source tree into the development container, making it possible to observe and test changes in real time. -``` -cd ~/projects -git clone git@github.com:ansible/galaxy.git -``` +### Prerequisites -Configure git with your name and email so that your commits are correctly associated with your GitHub account: +#### make -``` -cd ~/projects/galaxy/ -git config user.name "Joe Developer" -git config user.email "joe@ansibleworks.com" -``` +For convenience, many of the commands you'll be running to build, start, stop and interact with the development containers have been added to the project [Makefile](./Makefile). Unless you really like typing, you'll want to take advantage of this. Check that you have access to the `make` command, and if not, install the `make` package for your OS. + +#### Docker + +Prior to starting the development services, you'll need `docker` and `docker-compose`. On Linux, you can generally find these in your distro's packaging, but you may find that Docker themselves maintain a separate repo that tracks more closely to the latest releases. + +For macOS and Windows, we recommend [Docker for Mac](https://www.docker.com/docker-mac) and [Docker for Windows](https://www.docker.com/docker-windows) +respectively. + +For Linux platforms, refer to the following from Docker: + +**Fedora** + +> https://docs.docker.com/engine/installation/linux/docker-ce/fedora/ + +**Centos** + +> https://docs.docker.com/engine/installation/linux/docker-ce/centos/ + +**Ubuntu** + +> https://docs.docker.com/engine/installation/linux/docker-ce/ubuntu/ + +**Debian** + +> https://docs.docker.com/engine/installation/linux/docker-ce/debian/ + +**Arch** + +> https://wiki.archlinux.org/index.php/Docker + +### Build the environment -Start developing by first creating a feature branch. The simplest way is by using Git Flow: +#### Fork and clone the Galaxy repo +If you have not done so already, you'll need to fork the Galaxy repo on GitHub. For more on how to do this, see [Fork a Repo](https://help.github.com/articles/fork-a-repo/). + +#### Build the galaxy-dev image + +You're going to run a script that will build the`galaxy-dev` image, which will contain everything needed to run the Galaxy frontend web server, backend Django server, and Celery task runner. Prior to building this image, the script will first build the `galaxy-build` image, which contains all of the required OS packages, Python dependencies, and frontend tools and packages. The `galaxy-build` image is then used as the base image to create `galaxy-dev`. + +If you're curious about what actually goes into building the images, you'll find the Dockerfile for `galaxy-build` in [scripts/docker-release](./scripts/docker-release). The actual filename is `Dockerfile.build`. And you'll find the Dockerfile for `galaxy-dev` in [scripts/docker-dev](./scripts/docker-dev). + +Run the following to build the image: + +```bash +# Set your working directory to the project root +$ cd galaxy + +# Start the build process +$ make dev/build ``` -cd ~/projects/galaxy -git flow feature start mynewfeature + +Once the build completes, you will the `galaxy-dev` and `galaxy-build` images in your local image cache. Use the `docker images` command to check, as follows: + +```bash +$ docker images + +REPOSITORY TAG IMAGE ID CREATED SIZE +galaxy-dev latest 2f552729e204 22 seconds ago 748MB +galaxy-build latest c456f5c226d3 2 minutes ago 568MB +centos 7 196e0ce0c9fb 6 weeks ago 197MB ``` -### Build Galaxy +### Start the containers + +We use `docker-compose` to run the containers. If you're curious about the services, and and how they're configured, the compose file is [scripts/compose-dev.yml](./scripts/compose-dev.yml) + +#### Quick start + +There are a couple different ways to start the development containers. If all of this is new, and you just want to get things going, run the following command to start the containers in an attached mode. All that means is that the STDOUT and STDERR for each container will stream to the STDOUT and STDERR of your terminal session. Your session will be totally consumed with the output, which is OK. Seeing the output lets you know what's actually happening in the containers. To run other commands, and do other things from the your terminal, you'll need to start a second session. -You should already have Docker running and Ansible Container installed. To build the Galaxy images, run the following from the root directory of your Galaxy clone: +So without further ado, run the following: +```bash +# Set your working directory to the project root +$ cd galaxy + +# Start the build process +$ make dev/up ``` -$ make build + +Any missing images (i.e., postgresql, memcached, rabbitmq, elasticsearch) will be pulled, and the containers will launch. Aftr the above commands completes, you can take a look at the containers by running `docker ps` in your second terminal session. + +#### Running detached + +If you prefer to start the containers in detached mode, where they run in the background, run the following command: + +```bash +# Set your working directory to the project root +$ cd galaxy + +# Start the build process +$ make dev/up_detached ``` -**NOTE**: The build process downloads images from Docker Hub for most services, excluding *django* and *gulp*. The *django* and *gulp* services are built from a *centos:7* base image, and the build involves downloading and installing both *yum* and *pip* packages. The build can run from 10 minutes up to an hour depending on the speed of your internet connection, and the amount of available CPU and memory. +#### Running through tmux + +If you're familiar with `tmux`, and you would prefer to view the container output through a tmux session, use the following process to launch the containers, and then start the processes within the `galaxy` service using `tmux`. -After the build completes, you will see the following Docker images: +Start by executing the following command to launch the containers in detached mode, running in the background. Note that there will be no running processes in the `galaxy` service container, because you'll be starting them using `tmux` in the next step. +```bash +# Set your working directory to the project root +$ cd galaxy + +# Start the build process +$ make dev/up_tmux ``` -# View the Galaxy images -$ docker images -REPOSITORY TAG IMAGE ID CREATED SIZE -galaxy-django 20170717173749 b7f05cefadbe 23 minutes ago 872MB -galaxy-django latest b7f05cefadbe 23 minutes ago 872MB -galaxy-gulp latest 2461ebb38ac5 4 hours ago 405MB -galaxy-gulp 20170714223513 47702c4705ec 2 days ago 405MB -galaxy-conductor latest 21f24543e888 2 days ago 518MB -memcached latest 4b6d78556a83 10 days ago 82.6MB -centos 7 36540f359ca3 11 days ago 193MB -rabbitmq latest f8fb6fc4d6ff 3 weeks ago 124MB -elasticsearch 2.4.1 8e3cf79edcc3 8 months ago 346MB -postgres 9.5.4 2417ea518abc 8 months ago 264MB +Once the above commands completes, you can view the containers by running `docker ps`. The service we're most interested in is `galaxy`, and it's container name will be `galaxy_galaxy_1`. Before we can launch `tmux`, we need to wait for database migrations and other setup to complete. To see what's going on inside the `galaxy` service container, and whether or not the setup is complete, run the following to stream its logging output: + +``` +$ docker logs -f galaxy_galaxy_1 ``` -Start the services by running the following: +The above will stream the log output to your terminal window. When all the migrations and setup work is done, the output stream will stop, and you'll see output similar to the following: ``` -# Start the Galaxy services -$ make run + Applying socialaccount.0003_extra_data_default_dict... OK ++ make build_indexes +Rebuild Custom Indexes +/var/lib/galaxy/venv/bin/python ./manage.py rebuild_galaxy_indexes +/var/lib/galaxy/venv/lib/python2.7/site-packages/maintenance/middleware.py:3: RemovedInDjango19Warning: django.utils.importlib will be removed in Django 1.9. + from django.utils.importlib import import_module + +Rebuild Search Index +/var/lib/galaxy/venv/bin/python ./manage.py rebuild_index --noinput +/var/lib/galaxy/venv/lib/python2.7/site-packages/maintenance/middleware.py:3: RemovedInDjango19Warning: django.utils.importlib will be removed in Django 1.9. + from django.utils.importlib import import_module + +Removing all documents from your index because you said so. +All documents removed. +Indexing 0 roles ++ '[' 1 == 1 ']' ++ scripts/docker-dev/sleep.sh ``` -The *django* service executes the Django development web server and a celery process. Before starting the web server for the first time, it will perform database migrations, load test data, and rebuild the search indexes. These operations may take up to 10 minutes, so before attempting to access the web server, check the log file to see which task is being executed, and whether or not the web server has started. The following command will attach to the service's STDOUT and display the log output in realtime: +If you see the very last line, `scripts/docker-dev/sleep.sh`, you're ready to start `tmux` and launch the processes inside the `galaxy` service container. Press `Ctrl-c` to stop streaming the logging output, and then run the following: +```bash +# Set your working directory to the project root +$ cd galaxy + +# Start tmux +$ make dev/tmux ``` -# Follow the django service log output -$ docker logs -f galaxy_django_1 + +### Accessing the Galaxy web site + +After doing all this work, you're anxious to view your shiny new Galaxy site, aren't you? Well, not so fast. As discussed above in [Running through tmux](#running-through-tmux), you'll need to first check to make sure that all the database migrations and setup work completed, and processes are running inside the `galaxy` service container. + +Check the output stream from the `galaxy` service container, and look for the completion of database migrations, and the start of the `gulp` web server. If you see output similar to the following, then you know that `gulp` is running and accepting connections: + ``` +[03:10:00] Using gulpfile /galaxy/gulpfile.js +[03:10:01] Starting 'less'... +[03:10:01] Starting 'server'... +[HPM] Proxy created: / -> http://localhost:8888 +[03:10:01] Finished 'server' after 130 ms +[03:10:01] Starting 'watch'... +[03:10:03] Finished 'watch' after 2.06 s +[Browsersync] Access URLs: + ----------------------------------- + Local: http://localhost:8000 + External: http://172.18.0.6:8000 + ----------------------------------- + UI: http://localhost:3001 + UI External: http://172.18.0.6:3001 + ----------------------------------- +[Browsersync] Serving files from: /galaxy +[03:10:04] Finished 'less' after 3.23 s +[03:10:04] Starting 'default'... +[03:10:04] Finished 'default' after 108 μs +``` + +OK, go ahead. What are you waiting for? You can view your Galaxy site at: [http://localhost:8000](http://localhost:8000). -Once the web server starts, use Ctrl-C to stop following the log output, or switch to a different terminal session. +**NOTE:** -You can access the web server at [http://localhost:8000](http://localhost:8000). +>You won't be able to authenticate until you perform the post build steps for creating an `admin` user and configuring GitHub authentication. -### Create an admin user +### Post build setup -To create a superuser with access to the admin site, open a new terminal session or window, and run `make createsuperuser`. The following shows the creation of an admin user: +#### Create an admin user + +From the root of the project tree, run `make dev/createsuperuser` to start the creation process. You'll be prompted for the vital details as depicted below: ``` -$ make createsuperuser +$ make dev/createsuperuser Create Superuser Username: admin @@ -94,7 +218,8 @@ Password: Password (again): Superuser created successfully. ``` -The admin site can be accessed at [http://localhost:8000/galaxy__admin__site](http://localhost:8000/galaxy__admin__site). + +The Django admin site can be accessed at [http://localhost:8000/galaxy__admin__site](http://localhost:8000/galaxy__admin__site). ### Connect to GitHub @@ -110,67 +235,15 @@ After you save the new application, access your local Galaxy admin site at [http After logging into the admin site, you'll create a new social application. Start by finding `Social applications` at the bottom of the table, and clicking the *Add* button to its right. On the next screen, set the *provider* to `GitHub`, and enter `GitHub` as the *name*. From the new GitHub Oauth application you just created, copy the *ClientID* value into *Client id*, and copy the *Client Secret* value into *Secret key*. Under *Sites*, add `localhost` to *Chosen sites*. Save the changes. -Log out of the *admin* account, and go back to [http://localhost:8000(http://localhost:8000). Click the GitHub logo under `Log into Galaxy with GitHub`. On the next screen, you should see the message `Verify Your Email Address`. - -The email was written to a log file found on the *django* container. To access it, use the following commands to connect to the container and view the file: -``` -# Connect to the container and start the bash shell -$ docker exec -it ansible_django_1 /bin/bash - -# Set the working directory to /galaxy_logs/email -$ cd /galaxy_logs/email - -# List the log files -$ ls -l - -total 4 --rw-r--r-- 1 django root 863 Jan 19 05:04 20170119-000450-139972593475792.log - -# Show the contents of the latest message -$ cat 20170119-000450-139972593475792.log - -From: webmaster@localhost -To: foo@gmail.com -Date: Thu, 19 Jan 2017 05:04:50 -0000 -Message-ID: <20170119050450.181.62126@c164ede7ada7> - -You are receiving this email because someone (hopefully you) created an account at https://galaxy.ansible.com. To confirm this is correct, please click on the following link to verify your email address: - -http://localhost:8000/accounts/confirm-email/MQ:1cU4u2:SfDV-p07an0_hv5K9ggi1kuPbJM/ - -If you believe you have received this email in error, please disregard it. - -If you have any questions regarding this email, or if you have trouble with the link provided above, please contact support. - -Thanks! -``` -From the latest log file, retrieve the verification URL, and paste it into your browser. Once the page loads, click the `Confirm` button, and on the next page click the blue GitHub logo to log in. +Log out of the *admin* account, and go back to [http://localhost:8000(http://localhost:8000). Click the GitHub logo under `Log into Galaxy with GitHub`. ### Modifying static assets The Javascript, CSS and HTML components for the web site can be found in the [galaxy/static](./galaxy/static) folder. Within this folder, the *gulp* service watches for modifications to `less/*.less` stylesheets, and automatically recompiles the CSS and refreshes your browser. It also refreshes your browser whenever changes are made to `js/*/*.js` and `partion/*.html` files. -If changes to frontend assets are not being reflected in your browser, use the following command to check the gulp service logs: - -``` -# View the gulp service log -$ docker logs galaxy_gulp_1 -``` - ### Stop services -To stop all services, run `make stop`. You can then run `docker ps` to check the state of the services. - -### Other useful commands - -Review the Makefile for additional commands. Examples include: - -- `make build_indexes` to rebuild the search indexes -- `make clean` to remove all galaxy images, containers and logging data -- `make createsuperuser` to create the Django admin user -- `make migrate` to generate and apply Django migrations -- `make psql` to access the database command line tool -- `make refresh` to remove all galaxy images, containers, logging data, rebuild images, and restart the containers +To stop all services, run `make dev/down`. ### Submitting Code diff --git a/Makefile b/Makefile index cbe5852ea..9a9c2de00 100644 --- a/Makefile +++ b/Makefile @@ -1,8 +1,6 @@ PYTHON=python SITELIB=$(shell $(PYTHON) -c "from distutils.sysconfig import get_python_lib; print get_python_lib()") -# Get the branch information from git -GIT_DATE := $(shell git log -n 1 --format="%ai") DATE := $(shell date -u +%Y%m%d%H%M) VERSION=$(shell $(PYTHON) -c "from galaxy import __version__; print(__version__.split('-')[0])") @@ -28,83 +26,89 @@ DEB_BUILD_DIR=deb-build/galaxy-$(VERSION) DEB_PKG_RELEASE=$(VERSION)-$(RELEASE) endif -.PHONY: clean clean_dist clean_images clean_containers \ - refresh migrate migrate_empty makemigrations build_from_scratch \ - build build_debug \ - run run_production \ - flake8 test waitenv \ - build_indexes \ - sdist stop stop_production requirements ui_build export_test_data import_test_data createsuperuser \ - refresh_role_counts shell +VENV_BIN=/var/lib/galaxy/venv/bin -# Remove containers, images and ~/.galaxy -clean: - @./clean.sh +.PHONY: clean_dist \ + sdist requirements ui_build export_test_data import_test_data \ + refresh_role_counts -# Refresh development environment after pulling new code. -refresh: clean build run +.PHONY: dev/build +dev/build: + scripts/build-docker-dev.sh -# Remove containers -clean_containers: - ./clean.sh containers - -# Remove images -clean_images: - ./clean.sh images - -# Create and execute database migrations -migrate: - @docker exec -i -t galaxy_django_1 /venv/bin/python ./manage.py makemigrations main --noinput - @docker exec -i -t galaxy_django_1 /venv/bin/python ./manage.py migrate --noinput - -# Create an empty migration -migrate_empty: - @docker exec -i -t galaxy_django_1 /venv/bin/python ./manage.py makemigrations --empty main - -makemigrations: - @docker exec -i -t galaxy_django_1 /venv/bin/python ./manage.py makemigrations main - -psql: - @docker exec -i -t galaxy_django_1 psql -h postgres -d galaxy -U galaxy - -# Build Galaxy images -build: - ansible-container --debug --devel build - -# Rebuild Galaxy search indexes -build_indexes: - @echo "Rebuild Custom Indexes" - @docker exec -i -t galaxy_django_1 /venv/bin/python ./manage.py rebuild_galaxy_indexes - @echo "Rebuild Search Index" - @docker exec -i -t galaxy_django_1 /venv/bin/python ./manage.py rebuild_index --noinput - -createsuperuser: +.PHONY: dev/createsuperuser +dev/createsuperuser: @echo "Create Superuser" - @docker exec -i -t galaxy_django_1 /venv/bin/python ./manage.py createsuperuser - -# Run flake8 inside the django container -flake8: - docker run -v ${PWD}:/galaxy -w /galaxy galaxy-django:latest /venv/bin/flake8 --config=tox.ini galaxy - -test: - docker exec -i -t galaxy_django_1 /venv/bin/python ./manage.py test - -waitenv: - docker exec -i -t galaxy_django_1 /venv/bin/python ./manage.py waitenv - -# Start Galaxy containers with django and gulp in the foreground -run: - ansible-container --debug --devel run - -# Start production -run_production: - ansible-container --debug --devel run --production - -stop: - @ansible-container stop --force - -stop_production: - @ansible-container stop --force --production + @docker-compose -f ./scripts/compose-dev.yml -p galaxy exec galaxy bash -c '${VENV_BIN}/python ./manage.py createsuperuser' + +.PHONY: dev/migrate +dev/migrate: + docker-compose -f ./scripts/compose-dev.yml -p galaxy exec galaxy bash -c '${VENV_BIN}/python ./manage.py migrate --noinput' + +.PHONY: dev/makemigrations +dev/makemigrations: + docker-compose -f ./scripts/compose-dev.yml -p galaxy exec galaxy bash -c '${VENV_BIN}/python ./manage.py makemigrations main' + +.PHONY: dev/flake8 +dev/flake8: + @echo "Running flake8" + @docker-compose -f ./scripts/compose-dev.yml -p galaxy exec galaxy bash -c '${VENV_BIN}/flake8 --config=tox.ini galaxy' + +.PHONY: dev/test +dev/test: + @echo "Running tests" + @docker-compose -f ./scripts/compose-dev.yml -p galaxy exec galaxy bash -c '${VENV_BIN}/python ./manage.py test' + +.PHONY: dev/up +dev/up: + docker-compose -f ./scripts/compose-dev.yml -p galaxy up + +.PHONY: dev/up_detached +dev/up_detached: + # Start all containers detached + docker-compose -f ./scripts/compose-dev.yml -p galaxy up -d + +.PHONY: dev/up_tmux +dev/up_tmux: + # Run before dev/tmux to start containers detached and no processes running in the galaxy container. + TMUX=1 docker-compose -f ./scripts/compose-dev.yml -p galaxy up -d + +.PHONY: dev/down +dev/down: + docker-compose -f ./scripts/compose-dev.yml -p galaxy down + +.PHONY: dev/restart +dev/restart: + # Restart one or more services + docker-compose -f ./scripts/compose-dev.yml -p galaxy restart $(filter-out $@,$(MAKECMDGOALS)) + +.PHONY: dev/stop +dev/stop: + # Stop one or more services + docker-compose -f ./scripts/compose-dev.yml -p galaxy stop $(filter-out $@,$(MAKECMDGOALS)) + +.PHONY: dev/tmux_noattach +dev/tmux_noattach: + # Create the tmux session. Do NOT call directly. Use dev/tmux or dev/tmuxcc instead. + tmux new-session -d -s galaxy -n galaxy 'bash -c "make runserver; exec bash"' + tmux new-window -t galaxy:1 -n celery 'bash -c "make celery; exec bash"' + tmux new-window -t galaxy:2 -n gulp 'bash -c "make gulp; exec bash"' + tmux select-window -t galaxy:0 + +.PHONY: dev/tmux +dev/tmux: + # Connect to the galaxy container, start processes, and pipe stdout/stderr through a tmux session + docker-compose -f ./scripts/compose-dev.yml -p galaxy exec galaxy bash -c 'make dev/tmux_noattach; tmux -2 attach-session -t galaxy' + +.PHONY: dev/tmuxcc +dev/tmuxcc: dev/tmux_noattach + # Same as above using iTerm's built-in tmux support + docker-compose -f ./scripts/compose-dev.yml -p galaxy exec galaxy bash -c 'make dev/tmux_noattach; tmux -2 -CC attach-session -t galaxy' + +.PHONY: dev/gulp_build +dev/gulp_build: + # build UI components + docker-compose -f ./scripts/compose-dev.yml -p galaxy exec galaxy bash -c '/usr/local/bin/gulp build' sdist: clean_dist ui_build if [ "$(OFFICIAL)" = "yes" ] ; then \ @@ -113,15 +117,6 @@ sdist: clean_dist ui_build BUILD=$(BUILD) $(PYTHON) setup.py sdist_galaxy; \ fi -requirements: - @if [ "$(VIRTUAL_ENV)" ]; then \ - pip install distribute==0.7.3; \ - pip install -r requirements.txt; \ - $(PYTHON) fix_virtualenv_setuptools.py; \ - else \ - sudo pip install -r requirements.txt; \ - fi - clean_dist: rm -rf dist/* rm -rf build rpm-build *.egg-info @@ -129,9 +124,6 @@ clean_dist: rm -f galaxy/static/dist/*.js find . -type f -regex ".*\.py[co]$$" -delete -ui_build: - node node_modules/gulp/bin/gulp.js build - export_test_data: @echo Export data to test-data/role_data.dmp.gz @docker exec -i -t galaxy_django_1 /galaxy/test-data/export.sh @@ -144,7 +136,34 @@ refresh_role_counts: @echo Refresh role counts @docker exec -i -t galaxy_django_1 /venv/bin/python ./manage.py refresh_role_counts -shell: - @echo Starting shell on galaxy_django_1 - @docker exec -i -t galaxy_django_1 /bin/bash +.PHONY: celery +celery: + ${VENV_DIR}/bin/python manage.py celeryd -B --autoreload -Q 'celery,import_tasks,login_tasks' + +.PHONY: runserver +runserver: + ${VENV_DIR}/bin/python manage.py runserver 0.0.0.0:8888 + +.PHONY: gulp +gulp: + /usr/local/bin/gulp + +.PHONY: waitenv +waitenv: + @echo "Waiting for services to start..." + ${VENV_DIR}/bin/python ./manage.py waitenv + +.PHONY: migrate +migrate: + @echo "Run migrations" + ${VENV_DIR}/bin/python ./manage.py migrate --noinput + +.PHONY: build_indexes +build_indexes: + @echo "Rebuild Custom Indexes" + ${VENV_DIR}/bin/python ./manage.py rebuild_galaxy_indexes + @echo "Rebuild Search Index" + ${VENV_DIR}/bin/python ./manage.py rebuild_index --noinput +%: + @: diff --git a/galaxy/settings/default.py b/galaxy/settings/default.py index dc7ccf1e6..01428bb58 100644 --- a/galaxy/settings/default.py +++ b/galaxy/settings/default.py @@ -266,7 +266,7 @@ ACCOUNT_EMAIL_REQUIRED = True -ACCOUNT_EMAIL_VERIFICATION = 'mandatory' +ACCOUNT_EMAIL_VERIFICATION = 'none' ACCOUNT_EMAIL_SUBJECT_PREFIX = 'Ansible Galaxy ' @@ -326,7 +326,7 @@ # Logging # ========================================================= -LOGS_DIR = os.path.join(BASE_DIR, 'var', 'log') +# LOGS_DIR = os.path.join(BASE_DIR, 'var', 'log') # TODO(cutwater): Adjust logging config for production environment # TODO(cutwater): Review logging config @@ -356,30 +356,30 @@ 'filters': ['require_debug_false'], 'class': 'django.utils.log.AdminEmailHandler' }, - 'allauth_logfile': { - 'level': 'DEBUG', - 'class': 'logging.handlers.RotatingFileHandler', - 'filename': os.path.join(LOGS_DIR, 'allauth.log'), - 'maxBytes': 500000, - 'backupCount': 4, - 'formatter': 'verbose', - }, - 'django_logfile': { - 'level': 'DEBUG', - 'class': 'logging.handlers.RotatingFileHandler', - 'filename': os.path.join(LOGS_DIR, 'django.log'), - 'maxBytes': 500000, - 'backupCount': 4, - 'formatter': 'verbose', - }, - 'galaxy_logfile': { - 'level': 'DEBUG', - 'class': 'logging.handlers.RotatingFileHandler', - 'filename': os.path.join(LOGS_DIR, 'galaxy.log'), - 'maxBytes': 500000, - 'backupCount': 4, - 'formatter': 'verbose', - }, +# 'allauth_logfile': { +# 'level': 'DEBUG', +# 'class': 'logging.handlers.RotatingFileHandler', +# 'filename': os.path.join(LOGS_DIR, 'allauth.log'), +# 'maxBytes': 500000, +# 'backupCount': 4, +# 'formatter': 'verbose', +# }, +# 'django_logfile': { +# 'level': 'DEBUG', +# 'class': 'logging.handlers.RotatingFileHandler', +# 'filename': os.path.join(LOGS_DIR, 'django.log'), +# 'maxBytes': 500000, +# 'backupCount': 4, +# 'formatter': 'verbose', +# }, +# 'galaxy_logfile': { +# 'level': 'DEBUG', +# 'class': 'logging.handlers.RotatingFileHandler', +# 'filename': os.path.join(LOGS_DIR, 'galaxy.log'), +# 'maxBytes': 500000, +# 'backupCount': 4, +# 'formatter': 'verbose', +# }, 'console': { 'level': 'INFO', 'class': 'logging.StreamHandler', @@ -394,7 +394,7 @@ 'propagate': True, }, 'django': { - 'handlers': ['django_logfile'], + 'handlers': ['console'], 'level': 'DEBUG', 'propagate': True, }, diff --git a/galaxy/settings/development.py b/galaxy/settings/development.py index 200915f62..5ea366b04 100644 --- a/galaxy/settings/development.py +++ b/galaxy/settings/development.py @@ -110,7 +110,7 @@ # Celery settings # --------------------------------------------------------- -BROKER_URL = 'amqp://galaxy:galaxy@rabbit:5672/galaxy' +BROKER_URL = 'amqp://galaxy:galaxy@rabbitmq:5672/galaxy' # ========================================================= # Galaxy Settings @@ -122,7 +122,7 @@ WAIT_FOR = [ {'host': 'postgres', 'port': 5432}, - {'host': 'rabbit', 'port': 5672}, + {'host': 'rabbitmq', 'port': 5672}, {'host': 'memcache', 'port': 11211}, {'host': 'elastic', 'port': 9200} ] diff --git a/gulpfile.js b/gulpfile.js index 1bd5a4aba..927db3e42 100644 --- a/gulpfile.js +++ b/gulpfile.js @@ -12,7 +12,7 @@ const history = require('connect-history-api-fallback') gulp.task('server', function() { - const proxyDjango = proxy('/', {target: 'http://django:8000', xfwd: true}) + const proxyDjango = proxy('/', {target: 'http://localhost:8888', xfwd: true}) sync.init({ notify: false, diff --git a/container.yml b/old-scripts/container.yml similarity index 100% rename from container.yml rename to old-scripts/container.yml diff --git a/develop.yml b/old-scripts/develop.yml similarity index 100% rename from develop.yml rename to old-scripts/develop.yml diff --git a/fix_virtualenv_setuptools.py b/old-scripts/fix_virtualenv_setuptools.py similarity index 100% rename from fix_virtualenv_setuptools.py rename to old-scripts/fix_virtualenv_setuptools.py diff --git a/roles/README.md b/old-scripts/roles/README.md similarity index 100% rename from roles/README.md rename to old-scripts/roles/README.md diff --git a/roles/django-role/defaults/main.yml b/old-scripts/roles/django-role/defaults/main.yml similarity index 100% rename from roles/django-role/defaults/main.yml rename to old-scripts/roles/django-role/defaults/main.yml diff --git a/roles/django-role/files/common.sh b/old-scripts/roles/django-role/files/common.sh similarity index 100% rename from roles/django-role/files/common.sh rename to old-scripts/roles/django-role/files/common.sh diff --git a/roles/django-role/files/inventory b/old-scripts/roles/django-role/files/inventory similarity index 100% rename from roles/django-role/files/inventory rename to old-scripts/roles/django-role/files/inventory diff --git a/roles/django-role/files/start-development.sh b/old-scripts/roles/django-role/files/start-development.sh similarity index 100% rename from roles/django-role/files/start-development.sh rename to old-scripts/roles/django-role/files/start-development.sh diff --git a/roles/django-role/files/start.sh b/old-scripts/roles/django-role/files/start.sh similarity index 100% rename from roles/django-role/files/start.sh rename to old-scripts/roles/django-role/files/start.sh diff --git a/roles/django-role/files/update_site.sql b/old-scripts/roles/django-role/files/update_site.sql similarity index 100% rename from roles/django-role/files/update_site.sql rename to old-scripts/roles/django-role/files/update_site.sql diff --git a/roles/django-role/tasks/install-python-packages.yml b/old-scripts/roles/django-role/tasks/install-python-packages.yml similarity index 100% rename from roles/django-role/tasks/install-python-packages.yml rename to old-scripts/roles/django-role/tasks/install-python-packages.yml diff --git a/roles/django-role/tasks/main.yml b/old-scripts/roles/django-role/tasks/main.yml similarity index 100% rename from roles/django-role/tasks/main.yml rename to old-scripts/roles/django-role/tasks/main.yml diff --git a/roles/django-role/tasks/setup-galaxy.yml b/old-scripts/roles/django-role/tasks/setup-galaxy.yml similarity index 100% rename from roles/django-role/tasks/setup-galaxy.yml rename to old-scripts/roles/django-role/tasks/setup-galaxy.yml diff --git a/roles/django-role/templates/pgpass.j2 b/old-scripts/roles/django-role/templates/pgpass.j2 similarity index 100% rename from roles/django-role/templates/pgpass.j2 rename to old-scripts/roles/django-role/templates/pgpass.j2 diff --git a/roles/gulp-role/tasks/main.yml b/old-scripts/roles/gulp-role/tasks/main.yml similarity index 100% rename from roles/gulp-role/tasks/main.yml rename to old-scripts/roles/gulp-role/tasks/main.yml diff --git a/roles/nginx/AUTHORS b/old-scripts/roles/nginx/AUTHORS similarity index 100% rename from roles/nginx/AUTHORS rename to old-scripts/roles/nginx/AUTHORS diff --git a/roles/nginx/defaults/main.yml b/old-scripts/roles/nginx/defaults/main.yml similarity index 100% rename from roles/nginx/defaults/main.yml rename to old-scripts/roles/nginx/defaults/main.yml diff --git a/roles/nginx/files/nginx.repo b/old-scripts/roles/nginx/files/nginx.repo similarity index 100% rename from roles/nginx/files/nginx.repo rename to old-scripts/roles/nginx/files/nginx.repo diff --git a/roles/nginx/tasks/main.yml b/old-scripts/roles/nginx/tasks/main.yml similarity index 100% rename from roles/nginx/tasks/main.yml rename to old-scripts/roles/nginx/tasks/main.yml diff --git a/roles/nginx/templates/nginx.conf.j2 b/old-scripts/roles/nginx/templates/nginx.conf.j2 similarity index 100% rename from roles/nginx/templates/nginx.conf.j2 rename to old-scripts/roles/nginx/templates/nginx.conf.j2 diff --git a/scripts/build-docker-dev.sh b/scripts/build-docker-dev.sh new file mode 100755 index 000000000..606c3e665 --- /dev/null +++ b/scripts/build-docker-dev.sh @@ -0,0 +1,9 @@ +#!/bin/bash + +set -o nounset +set -o errexit + +docker build --rm -t galaxy-build -f scripts/docker-release/Dockerfile.build . + +docker build --rm -t galaxy-dev -f scripts/docker-dev/Dockerfile . + diff --git a/scripts/compose-dev.yml b/scripts/compose-dev.yml new file mode 100644 index 000000000..62d3a616c --- /dev/null +++ b/scripts/compose-dev.yml @@ -0,0 +1,37 @@ +version: '3' + +services: + galaxy: + image: galaxy-dev:latest + build: + context: ./docker-dev + volumes: + - ${PWD}:/galaxy:rw + environment: + - TMUX + ports: + - '8000:8000' + links: + - postgres + - elastic + - memcache + - rabbitmq + postgres: + image: postgres:9.5.4 + environment: + - POSTGRES_DB=galaxy + - POSTGRES_USER=galaxy + - POSTGRES_PASSWORD=galaxy + + elastic: + image: elasticsearch:2.4.1 + + memcache: + image: memcached:latest + + rabbitmq: + image: rabbitmq:latest + environment: + - RABBITMQ_DEFAULT_USER=galaxy + - RABBITMQ_DEFAULT_PASS=galaxy + - RABBITMQ_DEFAULT_VHOST=galaxy diff --git a/scripts/docker-dev/Dockerfile b/scripts/docker-dev/Dockerfile new file mode 100644 index 000000000..021ae4502 --- /dev/null +++ b/scripts/docker-dev/Dockerfile @@ -0,0 +1,18 @@ +FROM galaxy-build + +LABEL com.ansible.galaxy.description="Ansible Galaxy development" + +RUN yum -y install vim tmux make +RUN ${VENV_DIR}/bin/pip install honcho supervisor flake8 +RUN localedef -c -i en_US -f UTF-8 en_US.UTF-8 + +ENV DJANGO_SETTINGS_MODULE galaxy.settings.development +ENV C_FORCE_ROOT=1 +ENV LANG en_US.UTF-8 +ENV LANGUAGE en_US:en +ENV LC_ALL en_US.UTF-8 +ENV PYTHONIOENCODING=utf8 + +WORKDIR /galaxy +CMD /galaxy/scripts/docker-dev/start-develop.sh + diff --git a/scripts/docker-dev/Procfile b/scripts/docker-dev/Procfile new file mode 100644 index 000000000..e8391f278 --- /dev/null +++ b/scripts/docker-dev/Procfile @@ -0,0 +1,3 @@ +celery: make celery +runserver: make runserver +gulp: make gulp diff --git a/scripts/docker-dev/sleep.sh b/scripts/docker-dev/sleep.sh new file mode 100755 index 000000000..1545f9bfe --- /dev/null +++ b/scripts/docker-dev/sleep.sh @@ -0,0 +1,6 @@ +#!/bin/bash + +# Keep the container running without filling stdout +while true; do + sleep 1 +done diff --git a/scripts/docker-dev/start-develop.sh b/scripts/docker-dev/start-develop.sh new file mode 100755 index 000000000..29fc26d2e --- /dev/null +++ b/scripts/docker-dev/start-develop.sh @@ -0,0 +1,16 @@ +#!/bin/bash + +set -o xtrace +set -o errexit + +cd /galaxy + +make waitenv +make migrate +make build_indexes + +if [ "${TMUX}" == "1" ]; then + scripts/docker-dev/sleep.sh +else + ${VENV_DIR}/bin/honcho start -f "scripts/docker-dev/Procfile" +fi diff --git a/scripts/docker-release/Dockerfile.build b/scripts/docker-release/Dockerfile.build index 591846e64..9ff269883 100644 --- a/scripts/docker-release/Dockerfile.build +++ b/scripts/docker-release/Dockerfile.build @@ -23,7 +23,8 @@ RUN yum -y install epel-release \ COPY package.json /tmp/package.json COPY yarn.lock /tmp/yarn.lock RUN cd /tmp \ - && yarn install + && yarn install \ + && yarn global add gulp ENV PIP_NO_CACHE_DIR off diff --git a/var/email/.gitkeep b/var/email/.gitkeep deleted file mode 100644 index e69de29bb..000000000 diff --git a/var/log/.gitkeep b/var/log/.gitkeep deleted file mode 100644 index e69de29bb..000000000