Merge pull request #750 from c2corg/dev-env

Developer can now run lint and tests

Makefile

@@ -52,9 +52,10 @@ test: .build/venv/bin/nosetests template .build/dev-requirements.timestamp .buil
 .PHONY: lint
 lint: .build/venv/bin/flake8
 	.build/venv/bin/flake8 c2corg_api es_migration
+	@echo "Wonderful, python style is Ok! Here is a beer : 🍺"
 
 .PHONY: install
-install: .build/requirements.timestamp template
+install: .build/requirements.timestamp .build/dev-requirements.timestamp template
 
 .PHONY: template
 template: $(TEMPLATE_FILES)

README.md

@@ -1,186 +1,51 @@
-API Application for camptocamp.org v6
-=====================================
+# [camptocamp.org](https://www.camptocamp.org) API
 
-Requirements
-------------
+[![GitHub license](https://img.shields.io/github/license/c2corg/v6_api.svg)](https://github.com/c2corg/v6_api/blob/master/LICENSE) [![Build Status](https://travis-ci.org/c2corg/v6_api.svg?branch=master)](https://travis-ci.com/c2corg/v6_api) [![Total alerts](https://img.shields.io/lgtm/alerts/g/c2corg/v6_api.svg?logo=lgtm&logoWidth=18)](https://lgtm.com/projects/g/c2corg/v6_api/alerts/) [![Language grade: Python](https://img.shields.io/lgtm/grade/python/g/c2corg/v6_api.svg?logo=lgtm&logoWidth=18)](https://lgtm.com/projects/g/c2corg/v6_api/context:javascript) [![Codacy Badge](https://api.codacy.com/project/badge/Grade/56217935d9cd43458eb5539ce52a8628)](https://app.codacy.com/app/c2corg/v6_api?utm_source=github.com&utm_medium=referral&utm_content=c2corg/v6_api&utm_campaign=Badge_Grade_Dashboard) [![Known Vulnerabilities](https://snyk.io/test/github/c2corg/v6_api/badge.svg)](https://snyk.io/test/github/c2corg/v6_api)
 
- * Python 3.4
- * PostgreSQL 9.4 / PostGIS 2.1
- * virtualenv (1.7 or higher)
- * ElasticSearch 2.3.2
- * Redis 2.8
- * GEOS (3.4 or higher)
+## Development environment
+On any OS, install [git](https://git-scm.com/) and [docker](https://docs.docker.com/install/). Then :
 
-On Debian/Ubuntu systems the required components may be installed using
+### Install
 
-    sudo apt-get install virtualenv python3-dev postgresql-9.4 postgresql-9.4-postgis-2.1 postgresql-contrib-9.4 libpq-dev libgeos-dev redis-server libffi6 libffi-dev
+```sh
+# Download camptocamp.org source code :
+git clone https://github.com/c2corg/v6_api
+cd v6_api
 
-The Debian package for ElasticSearch can be downloaded from https://www.elastic.co/downloads/past-releases/elasticsearch-2-3-2
+# Build dev env. It's quite long, but you have to do it once.
+docker-compose build
+```
 
-Checkout
---------
+### Run
 
-    git clone https://github.com/c2corg/v6_api.git
+```sh
+docker-compose up
+```
 
-Build
------
+:heart: <http://localhost:6543> :heart:
 
-    cd v6_api
-    make -f config/{user} install
+Press CTRL+C to terminate it.
 
-To set up the database
-----------------------
+### Check code quality
 
-    scripts/create_user_db.sh
+In another terminal (`docker-compose up` must be running) :
 
-If you want to specify a specific string as user, you can instead use:
+```sh
+docker-compose exec api make -f config/docker-dev lint
+```
 
-    USER=something scripts/create_user_db.sh
+### Run test suite 
 
-To set up ElasticSearch
-----------------------
+In another terminal (`docker-compose up` must be running) :
 
-    .build/venv/bin/initialize_c2corg_api_es development.ini
+```sh
+# First, inititate test database, must be done once
+docker-compose exec postgresql /v6_api/scripts/create_user_db_test.sh
 
-Run the migration script
-------------------------
+# Then
+docker-compose exec api make -f config/docker-dev test
+```
 
-See [/c2corg_api/scripts/migration/README.md](/c2corg_api/scripts/migration/README.md).
-
-
-Run the application
--------------------
-
-The API consists of three applications, the actual web-application, a syncer script
-that synchronizes the database with ElasticSearch and a background jobs script that
-purges the database of non activated accounts and expired tokens. The three have to
-be started separately.
-
-To start the background jobs script:
-
-    make -f config/$USER run-background-jobs
-
-To start the syncer script:
-
-    make -f config/$USER run-syncer
-
-In production, the scripts should be started using
-
-    make -f config/$USER run-background-jobs-prod
-    make -f config/$USER run-syncer-prod
-
-To start the web-application:
-
-    make -f config/$USER serve
-
-Open your browser at http://localhost:6543/ or http://localhost:6543/?debug (debug mode). Make sure you are
-using the port that is set in `config/$USER`.
-
-Available actions may be listed using:
-
-    make help
-
-Requests examples
------------------
-
-Get the list of waypoints:
-
-    GET http://localhost:6543/waypoints
-
-Get waypoint with id=1:
-
-    GET http://localhost:6543/waypoints/1
-
-Insert a waypoint:
-
-    curl -X POST -v \
-    -H "Content-Type: application/json" \
-    -d '{"waypoint_type": "summit", "elevation": 3779, "geometry": {"geom": "{\"type\": \"Point\", \"coordinates\": [635956, 5723604]}"},"locales": [{"lang": "fr", "title": "Mont Pourri"}]}' \
-    http://localhost:6543/waypoints
-
-Updating a waypoint:
-
-    curl -X PUT -v \
-    -H "Content-Type: application/json" \
-    -d '{"message": "Comment about change", "document": {"elevation": 4633, "maps_info": null, "version": 1, "document_id": 1, "waypoint_type": "summit", "locales": [{"lang": "fr", "version": 1, "title": "Mont Rose", "access": null, "description": null}]}}' \
-    http://localhost:6543/waypoints/1
-
-API documentation
------------------
-
-- [Simple search](./c2corg_api/views/search.py)
-- [Guidebook API](./c2corg_api/views/waypoint.py)
-
-Forum integration (discourse)
---------------------------
-
-See https://github.com/c2corg/v6_forum
-
-
-Run the tests
---------------
-
-Create a database that will be used to run the tests:
-
-    scripts/create_user_db_test.sh
-
-If you want to specify a specific string as user you can instead enter:
-
-    USER=something scripts/create_user_db_test.sh
-
-Then run the tests with:
-
-    make -f config/$USER test
-
-Or with the `check` target, which runs `flake8` and `test`:
-
-    make -f config/$USER check
-
-To run a specific test:
-
-    .build/venv/bin/nosetests c2corg_api/tests/views/test_waypoint.py
-    .build/venv/bin/nosetests -s  c2corg_api/tests/views/test_waypoint.py:TestWaypointRest.test_get_lang
-
-To see the debug output:
-
-    .build/venv/bin/nosetests -s
-
-
-Production checks
------------------
-
-Check the Discourse forum is up and available to the client API:
-
-    scripts/check_discourse_connection.sh [id]
-
-"id" is a v6 user id. The script should return a 200OK.
-
-
-Health service
---------------
-
-To get information about a running API instance, e.g. its version and the
-status of its components, the health service `/health` can be used:
-
-    GET http://localhost:6543/health
-
-
-Developer Tips
---------------
-
-The API is mainly built using the following components:
-* Pyramid (Python framework) http://docs.pylonsproject.org/en/latest/
-* SQLAlchemy (ORM) http://docs.sqlalchemy.org/en/rel_1_0/
-* Cornice (REST framework) https://cornice.readthedocs.org/en/latest/
-* Colander (validating and deserializing data) http://docs.pylonsproject.org/projects/colander/en/latest/
-
-Debugger toolbar
-----------------
-
-In case of unexpected exceptions, an error handler makes sure that a JSON
-response is returned. During development it can be convenient to see
-the Pyramid debugger toolbar in case of errors. To disable the error handler,
-add the following line to your config `config/{user}`:
-
-    export show_debugger_for_errors = true
+## Useful links in [wiki](https://github.com/c2corg/v6_api/wiki)
 
+[Full info about development environment](https://github.com/c2corg/v6_api/wiki/Development-environment-on-Linux)

config/dev

@@ -11,6 +11,7 @@ export tests_elasticsearch_index = c2corg_${instanceid}_tests
 
 export redis_db_queue = 6
 export redis_db_cache = 7
+export redis_url = memory:///
 
 # in case of unexpected errors, show the debug toolbar?
 export show_debugger_for_errors = false

config/docker-dev

@@ -0,0 +1,19 @@
+instanceid ?= dev
+base_url ?= /${instanceid}
+ui_url ?= http://localhost:6553
+
+include Makefile
+
+export db_name = c2corg_${instanceid}
+export tests_db_host = postgresql
+export tests_db_name = c2corg_tests
+export elasticsearch_index = c2corg_${instanceid}
+export tests_elasticsearch_host = elasticsearch
+export tests_elasticsearch_index = c2corg_${instanceid}_tests
+
+export redis_db_queue = 6
+export redis_db_cache = 7
+export redis_url = redis://redis:6379/
+
+# in case of unexpected errors, show the debug toolbar?
+export show_debugger_for_errors = false

dev_api.Dockerfile

@@ -0,0 +1,55 @@
+FROM docker.io/debian:jessie
+
+ENV DEBIAN_FRONTEND noninteractive
+
+ENV LC_ALL en_US.UTF-8
+
+RUN set -x \
+ && apt-get update \
+ && apt-get -y upgrade \
+ && apt-get -y --no-install-recommends install \
+    python3 \
+    python3-chardet \
+    python3-colorama \
+    python3-html5lib \
+    python3-pkg-resources \
+    python3-requests \
+    python3-six \
+    python3-urllib3 \
+    libgeos-c1 \
+    libpq5 \
+    libffi6 \
+    make \
+    vim \
+    sudo \
+    python3-dev \
+    python3-pip \
+    libgeos-dev \
+    libffi-dev \
+    libpq-dev \
+    virtualenv \
+    gcc \
+    git \
+    locales \
+ && echo "en_US.UTF-8 UTF-8" > /etc/locale.gen \
+ && locale-gen en_US.UTF-8 \
+ && dpkg-reconfigure locales \
+ && /usr/sbin/update-locale LANG=en_US.UTF-8
+
+COPY ./ /var/www/
+
+WORKDIR /var/www/
+
+RUN set -x \
+ && make -f config/docker-dev install \ 
+ && py3compile -f -X '^.*gevent/_util_py2.py$' .build/venv/
+
+ENV version="{version}" \
+    PATH=/var/www/.build/venv/bin/:$PATH
+
+COPY /docker-entrypoint.sh /
+COPY /docker-entrypoint.d/* /docker-entrypoint.d/
+ENTRYPOINT ["/docker-entrypoint.sh"]
+
+EXPOSE 8080
+CMD ["gunicorn", "--paste", "production.ini", "-u", "www-data", "-g", "www-data", "-b", ":8080"]

dev_db.Dockerfile

@@ -0,0 +1,19 @@
+FROM docker.io/c2corg/c2corg_pgsql:anon-2018-11-02
+
+RUN apt-get update
+RUN apt-get -y --no-install-recommends install sudo
+
+RUN export PGDATA=/c2corg_anon
+COPY ./docker-compose/pgsql-settings.d/ /c2corg_anon/pgsql-settings.d/
+
+COPY ./ /v6_api/
+
+# HOW to run a script on db in docker file ?????????
+
+# ADD ./scripts/create_user_db_test.sh /docker-entrypoint-initdb.d/ZZZ-06-create-user-db-test.sh
+
+# RUN sudo service postgresql start
+# RUN while !</dev/tcp/db/5432; do sleep 1; echo 'Wait'; done;
+# RUN /v6_api/scripts/create_user_db_test.sh
+# RUN sudo service postgresql stop
+