diff --git a/.dockerignore b/.dockerignore index 63ba87096..5c496c43b 100644 --- a/.dockerignore +++ b/.dockerignore @@ -1,5 +1,9 @@ backup/** +*.pyc container_data .cache node_modules OpenOversight/app/static/dist/* +/OpenOversight/tests/resources/.minio.sys/ +__pycache__ + diff --git a/.gitignore b/.gitignore index da5707c66..779558f10 100644 --- a/.gitignore +++ b/.gitignore @@ -96,4 +96,5 @@ vagrant/puppet/.tmp node_modules/ OpenOverSight/app/static/dist/ - +/OpenOversight/tests/resources/.minio.sys/ +OpenOversight/tests/resources/openoversight-test/ diff --git a/CONTRIB.md b/CONTRIB.md index 14f92168c..ccec299f9 100644 --- a/CONTRIB.md +++ b/CONTRIB.md @@ -56,17 +56,23 @@ Once you're done, `make stop` and `make clean` to stop and remove the containers ## Testing S3 Functionality We use an S3 bucket for image uploads. If you are working on functionality involving image uploads, -then you should follow the "S3 Image Hosting" section in [DEPLOY.md](/DEPLOY.md) to make a test S3 bucket -on Amazon Web Services. +you can utilize a local S3 compatible storage layer named Minio. +Test resources are located in `OpenOversight/tests/resources/openoversight-test`, note that `openoversight-test` +in this instance is the name of our local S3 "bucket". -Once you have done this, you can put your AWS credentials in the following environmental variables: +For testing in a staging / production environment, then you should follow the "S3 Image Hosting" section +in [DEPLOY.md](/DEPLOY.md) to make a test S3 bucket on Amazon Web Services. + +Once you have done this, you can update the `.env` file in the root of this repository, e.g. +*IMPORTANT NOTE: DO NOT COMMIT ANY CHANGES TO THE `.env` FILE, INCLUDING ANY AWS CREDENTIALS OR PASSWORDS.* ```sh -$ export S3_BUCKET_NAME=openoversight-test -$ export AWS_ACCESS_KEY_ID=testtest -$ export AWS_SECRET_ACCESS_KEY=testtest -$ export AWS_DEFAULT_REGION=us-east-1 +S3_BUCKET_NAME=openoversight-test +AWS_ACCESS_KEY_ID=minio123 +AWS_SECRET_ACCESS_KEY=minio123 +AWS_DEFAULT_REGION=us-east-1 ``` +*IMPORTANT NOTE: DO NOT COMMIT ANY CHANGES TO THE `.env` FILE, INCLUDING ANY AWS CREDENTIALS OR PASSWORDS.* Now when you run `make dev` as usual in the same session, you will be able to submit images to your test bucket. diff --git a/Makefile b/Makefile index 29c195f58..b5b01b824 100644 --- a/Makefile +++ b/Makefile @@ -41,12 +41,12 @@ populate: create_db ## Build and run containers test: start ## Run tests if [ -z "$(name)" ]; then \ if [ "$$(uname)" == "Darwin" ]; then \ - FLASK_ENV=testing docker-compose run --rm web pytest --doctest-modules -n $$(sysctl -n hw.logicalcpu) --dist=loadfile -v tests/ app; \ + FLASK_ENV=testing docker-compose run --rm web /bin/bash -c 'scripts/minio_forward.sh & pytest --doctest-modules -n "$$(sysctl -n hw.logicalcpu)" --dist=loadfile -v tests/ app'; \ else \ - FLASK_ENV=testing docker-compose run --rm web pytest --doctest-modules -n $$(nproc --all) --dist=loadfile -v tests/ app; \ + FLASK_ENV=testing docker-compose run --rm web /bin/bash -c 'scripts/minio_forward.sh & pytest --doctest-modules -n "$$(nproc --all)" --dist=loadfile -v tests/ app'; \ fi; \ else \ - FLASK_ENV=testing docker-compose run --rm web pytest --doctest-modules -v tests/ app -k $(name); \ + FLASK_ENV=testing docker-compose run --rm web /bin/bash -c 'scripts/minio_forward.sh & pytest --doctest-modules -v tests/ app -k $(name)'; \ fi .PHONY: lint diff --git a/OpenOversight/app/config.py b/OpenOversight/app/config.py index ad62a2971..597c8d6db 100644 --- a/OpenOversight/app/config.py +++ b/OpenOversight/app/config.py @@ -54,6 +54,7 @@ class DevelopmentConfig(BaseConfig): SQLALCHEMY_DATABASE_URI = os.environ.get('SQLALCHEMY_DATABASE_URI') NUM_OFFICERS = 15000 SITEMAP_URL_SCHEME = 'http' + S3_ENDPOINT = os.getenv('S3_ENDPOINT') class TestingConfig(BaseConfig): @@ -63,6 +64,7 @@ class TestingConfig(BaseConfig): NUM_OFFICERS = 120 APPROVE_REGISTRATIONS = False SITEMAP_URL_SCHEME = 'http' + S3_ENDPOINT = os.getenv('S3_ENDPOINT') class ProductionConfig(BaseConfig): diff --git a/OpenOversight/app/utils.py b/OpenOversight/app/utils.py index 1519887cd..2440f2744 100644 --- a/OpenOversight/app/utils.py +++ b/OpenOversight/app/utils.py @@ -6,6 +6,7 @@ from io import BytesIO import boto3 +from botocore.client import Config from botocore.exceptions import ClientError import botocore import datetime @@ -33,6 +34,13 @@ SAVED_UMASK = os.umask(0o077) +def get_s3_connection(config=None): + if current_app.config['S3_ENDPOINT']: + return boto3.client('s3', config=config, endpoint_url=current_app.config['S3_ENDPOINT']) + else: + return boto3.client('s3', config=config) + + def set_dynamic_default(form_field, value): # First we ensure no value is set already if not form_field.data: @@ -226,8 +234,22 @@ def compute_hash(data_to_hash): return hashlib.sha256(data_to_hash).hexdigest() +def get_presigned_image_url(s3_path: str): + unsigned = Config(signature_version=botocore.UNSIGNED) + s3_client_unsigned = get_s3_connection(unsigned) + + url = s3_client_unsigned.generate_presigned_url( + 'get_object', + Params={'Bucket': current_app.config['S3_BUCKET_NAME'], 'Key': s3_path} + ) + return url + + def upload_obj_to_s3(file_obj, dest_filename): - s3_client = boto3.client('s3') + if current_app.config['S3_ENDPOINT']: + s3_client = get_s3_connection(Config(signature_version='s3v4')) + else: + s3_client = get_s3_connection() # Folder to store files in on S3 is first two chars of dest_filename s3_folder = dest_filename[0:2] @@ -240,16 +262,7 @@ def upload_obj_to_s3(file_obj, dest_filename): current_app.config['S3_BUCKET_NAME'], s3_path, ExtraArgs={'ContentType': s3_content_type, 'ACL': 'public-read'}) - - config = s3_client._client_config - config.signature_version = botocore.UNSIGNED - url = boto3.resource( - 's3', config=config).meta.client.generate_presigned_url( - 'get_object', - Params={'Bucket': current_app.config['S3_BUCKET_NAME'], - 'Key': s3_path}) - - return url + return get_presigned_image_url(s3_path) def filter_by_form(form_data, officer_query, department_id=None): diff --git a/OpenOversight/scripts/entrypoint.sh b/OpenOversight/scripts/entrypoint.sh index 677931b40..7bf938374 100755 --- a/OpenOversight/scripts/entrypoint.sh +++ b/OpenOversight/scripts/entrypoint.sh @@ -2,6 +2,8 @@ if [ "${DOCKER_BUILD_ENV:-}" == "production" ]; then flask run --host=0.0.0.0 --port=3000 else + # forwarding port 9000 to minio for local S3 bucket + scripts/minio_forward.sh & yarn build yarn watch & flask run --host=0.0.0.0 --port=3000 diff --git a/OpenOversight/scripts/minio_forward.sh b/OpenOversight/scripts/minio_forward.sh new file mode 100755 index 000000000..67fd8e5e1 --- /dev/null +++ b/OpenOversight/scripts/minio_forward.sh @@ -0,0 +1 @@ +ncat -k -l localhost 9000 --sh-exec "ncat minio 9000" diff --git a/OpenOversight/tests/conftest.py b/OpenOversight/tests/conftest.py index 7e3b951bb..f88ab5e3a 100644 --- a/OpenOversight/tests/conftest.py +++ b/OpenOversight/tests/conftest.py @@ -16,7 +16,7 @@ import os from PIL import Image as Pimage -from OpenOversight.app import create_app, models +from OpenOversight.app import create_app, models, utils from OpenOversight.app.utils import merge_dicts from OpenOversight.app.models import db as _db, Unit, Job, Officer from OpenOversight.tests.routes.route_helpers import ADMIN_EMAIL, ADMIN_PASSWORD @@ -291,8 +291,13 @@ def add_mockdata(session): session.add_all(test_units) session.commit() - test_images = [models.Image(filepath='/static/images/test_cop{}.png'.format(x + 1), department_id=1) for x in range(5)] + \ - [models.Image(filepath='/static/images/test_cop{}.png'.format(x + 1), department_id=2) for x in range(5)] + test_images = [ + models.Image( + filepath=utils.get_presigned_image_url('/images/test_cop{}.png'.format(x + 1)), + department_id=1) for x in range(5)] + \ + [models.Image( + filepath=utils.get_presigned_image_url('/images/test_cop{}.png'.format(x + 1)), + department_id=2) for x in range(5)] test_officer_links = [ models.Link( @@ -307,7 +312,7 @@ def add_mockdata(session): author='the internet'), ] - officers = [generate_officer() for o in range(NUM_OFFICERS)] + officers = [generate_officer() for _ in range(NUM_OFFICERS)] officers[0].links = test_officer_links session.add_all(officers) session.add_all(test_images) diff --git a/OpenOversight/app/static/images/test_cop1.png b/OpenOversight/tests/resources/openoversight-test/images/test_cop1.png similarity index 100% rename from OpenOversight/app/static/images/test_cop1.png rename to OpenOversight/tests/resources/openoversight-test/images/test_cop1.png diff --git a/OpenOversight/app/static/images/test_cop2.png b/OpenOversight/tests/resources/openoversight-test/images/test_cop2.png similarity index 100% rename from OpenOversight/app/static/images/test_cop2.png rename to OpenOversight/tests/resources/openoversight-test/images/test_cop2.png diff --git a/OpenOversight/app/static/images/test_cop3.png b/OpenOversight/tests/resources/openoversight-test/images/test_cop3.png similarity index 100% rename from OpenOversight/app/static/images/test_cop3.png rename to OpenOversight/tests/resources/openoversight-test/images/test_cop3.png diff --git a/OpenOversight/app/static/images/test_cop4.png b/OpenOversight/tests/resources/openoversight-test/images/test_cop4.png similarity index 100% rename from OpenOversight/app/static/images/test_cop4.png rename to OpenOversight/tests/resources/openoversight-test/images/test_cop4.png diff --git a/OpenOversight/app/static/images/test_cop5.png b/OpenOversight/tests/resources/openoversight-test/images/test_cop5.png similarity index 100% rename from OpenOversight/app/static/images/test_cop5.png rename to OpenOversight/tests/resources/openoversight-test/images/test_cop5.png diff --git a/docker-compose.yml b/docker-compose.yml index d99292fed..071aae473 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -5,41 +5,60 @@ volumes: driver: local services: - postgres: - restart: always - image: postgres:latest - environment: - POSTGRES_USER: openoversight - POSTGRES_PASSWORD: terriblepassword - POSTGRES_DB: openoversight-dev - volumes: - - postgres:/var/lib/postgresql/data - ports: - - "5432:5432" + postgres: + restart: always + image: postgres:latest + environment: + POSTGRES_USER: openoversight + POSTGRES_PASSWORD: terriblepassword + POSTGRES_DB: openoversight-dev + volumes: + - postgres:/var/lib/postgresql/data + ports: + - "5432:5432" - web: - restart: always - build: + web: + restart: always + build: context: . args: - - DOCKER_BUILD_ENV - - TRAVIS_PYTHON_VERSION + - DOCKER_BUILD_ENV + - TRAVIS_PYTHON_VERSION dockerfile: ./dockerfiles/web/Dockerfile - environment: - AWS_ACCESS_KEY_ID: "${AWS_ACCESS_KEY_ID}" - AWS_SECRET_ACCESS_KEY: "${AWS_SECRET_ACCESS_KEY}" - AWS_DEFAULT_REGION: "${AWS_DEFAULT_REGION}" - S3_BUCKET_NAME: "${S3_BUCKET_NAME}" - APPROVE_REGISTRATIONS: "${APPROVE_REGISTRATIONS}" - FLASK_APP: app - FLASK_ENV: "${FLASK_ENV:-development}" - volumes: - - ./OpenOversight/:/usr/src/app/OpenOversight/:z - user: "${UID:?Docker-compose needs UID set to the current user id number. Try 'export UID' and run docker-compose again}" - links: - - postgres:postgres - expose: - - "3000" - command: scripts/entrypoint.sh - ports: - - "3000:3000" + environment: + AWS_ACCESS_KEY_ID: "${AWS_ACCESS_KEY_ID:-minio123}" + AWS_SECRET_ACCESS_KEY: "${AWS_SECRET_ACCESS_KEY:-minio123}" + S3_ENDPOINT: "${S3_ENDPOINT:-http://localhost:9000}" + AWS_DEFAULT_REGION: "${AWS_DEFAULT_REGION:-us-east-1}" + S3_BUCKET_NAME: "${S3_BUCKET_NAME:-openoversight-test}" + APPROVE_REGISTRATIONS: "${APPROVE_REGISTRATIONS}" + FLASK_APP: app + FLASK_ENV: "${FLASK_ENV:-development}" + volumes: + - ./OpenOversight/:/usr/src/app/OpenOversight/:z + user: "${UID:?Docker-compose needs UID set to the current user id number. Try 'export UID' and run docker-compose again}" + command: scripts/entrypoint.sh + ports: + - "3000:3000" + + minio: + image: minio/minio + environment: + MINIO_ACCESS_KEY: "${AWS_ACCESS_KEY_ID:-minio123}" + MINIO_SECRET_KEY: "${AWS_SECRET_ACCESS_KEY:-minio123}" + command: server /data + volumes: + - ./OpenOversight/tests/resources/:/data:rw + ports: + - "9000:9000" + + mc: + image: minio/mc + depends_on: + - minio + entrypoint: > + /bin/sh -c " + sleep 10; + /usr/bin/mc config host add minio http://minio:9000 ${AWS_ACCESS_KEY_ID:-minio123} ${AWS_SECRET_ACCESS_KEY:-minio123}; + /usr/bin/mc policy set public minio/${S3_BUCKET_NAME:-openoversight-test}; + " diff --git a/dockerfiles/web/Dockerfile b/dockerfiles/web/Dockerfile index b9f827de6..e7954caf0 100644 --- a/dockerfiles/web/Dockerfile +++ b/dockerfiles/web/Dockerfile @@ -11,7 +11,7 @@ ENV DISPLAY=:1 # install apt dependencies RUN echo "deb http://deb.debian.org/debian stretch-backports main" > /etc/apt/sources.list.d/backports.list -RUN apt-get update && apt-get install -y xvfb firefox-esr libpq-dev python3-dev && \ +RUN apt-get update && apt-get install -y xvfb firefox-esr libpq-dev python3-dev ncat && \ apt-get install -y -t stretch-backports libsqlite3-0 && apt-get clean # install node diff --git a/requirements.txt b/requirements.txt index 0db5d2719..7cadf5279 100644 --- a/requirements.txt +++ b/requirements.txt @@ -2,6 +2,7 @@ Flask==1.0.2 python-dotenv==0.7.1 boto3==1.5.10 werkzeug==1.0.1 +wtforms==2.3.3 Flask-Bootstrap==3.3.7.1 Flask-Limiter==1.0.1 Flask-Login==0.4.1