171/s3 local

.dockerignore

@@ -1,5 +1,9 @@
 backup/**
+*.pyc
 container_data
 .cache
 node_modules
 OpenOversight/app/static/dist/*
+/OpenOversight/tests/resources/.minio.sys/
+__pycache__
+

.gitignore

@@ -96,4 +96,5 @@ vagrant/puppet/.tmp
 node_modules/
 OpenOverSight/app/static/dist/
 
-
+/OpenOversight/tests/resources/.minio.sys/
+OpenOversight/tests/resources/openoversight-test/

CONTRIB.md

@@ -56,17 +56,23 @@ Once you're done, `make stop` and `make clean` to stop and remove the containers
 ## Testing S3 Functionality
 
 We use an S3 bucket for image uploads. If you are working on functionality involving image uploads,
-then you should follow the "S3 Image Hosting" section in [DEPLOY.md](/DEPLOY.md) to make a test S3 bucket
-on Amazon Web Services.
+you can utilize a local S3 compatible storage layer named Minio.
+Test resources are located in `OpenOversight/tests/resources/openoversight-test`, note that `openoversight-test`
+in this instance is the name of our local S3 "bucket".
 
-Once you have done this, you can put your AWS credentials in the following environmental variables:
+For testing in a staging / production environment, then you should follow the "S3 Image Hosting" section
+in [DEPLOY.md](/DEPLOY.md) to make a test S3 bucket on Amazon Web Services.
+
+Once you have done this, you can update the `.env` file in the root of this repository, e.g.
+*IMPORTANT NOTE: DO NOT COMMIT ANY CHANGES TO THE `.env` FILE, INCLUDING ANY AWS CREDENTIALS OR PASSWORDS.*
 
 ```sh
-$ export S3_BUCKET_NAME=openoversight-test
-$ export AWS_ACCESS_KEY_ID=testtest
-$ export AWS_SECRET_ACCESS_KEY=testtest
-$ export AWS_DEFAULT_REGION=us-east-1
+S3_BUCKET_NAME=openoversight-test
+AWS_ACCESS_KEY_ID=minio123
+AWS_SECRET_ACCESS_KEY=minio123
+AWS_DEFAULT_REGION=us-east-1
 ```
+*IMPORTANT NOTE: DO NOT COMMIT ANY CHANGES TO THE `.env` FILE, INCLUDING ANY AWS CREDENTIALS OR PASSWORDS.*
 
 Now when you run `make dev` as usual in the same session, you will be able to submit images to
 your test bucket.

Makefile

@@ -41,12 +41,12 @@ populate: create_db  ## Build and run containers
 test: start  ## Run tests
 	if [ -z "$(name)" ]; then \
 	    if [ "$$(uname)" == "Darwin" ]; then \
-			FLASK_ENV=testing docker-compose run --rm web pytest --doctest-modules -n $$(sysctl -n hw.logicalcpu) --dist=loadfile -v tests/ app; \
+			FLASK_ENV=testing docker-compose run --rm web /bin/bash -c 'scripts/minio_forward.sh & pytest --doctest-modules -n "$$(sysctl -n hw.logicalcpu)" --dist=loadfile -v tests/ app'; \
 		else \
-			FLASK_ENV=testing docker-compose run --rm web pytest --doctest-modules -n $$(nproc --all) --dist=loadfile -v tests/ app; \
+			FLASK_ENV=testing docker-compose run --rm web /bin/bash -c 'scripts/minio_forward.sh & pytest --doctest-modules -n "$$(nproc --all)" --dist=loadfile -v tests/ app'; \
 		fi; \
 	else \
-	    FLASK_ENV=testing docker-compose run --rm web pytest --doctest-modules -v tests/ app -k $(name); \
+	    FLASK_ENV=testing docker-compose run --rm web /bin/bash -c 'scripts/minio_forward.sh & pytest --doctest-modules -v tests/ app -k $(name)'; \
 	fi
 
 .PHONY: lint

OpenOversight/app/config.py

@@ -54,6 +54,7 @@ class DevelopmentConfig(BaseConfig):
     SQLALCHEMY_DATABASE_URI = os.environ.get('SQLALCHEMY_DATABASE_URI')
     NUM_OFFICERS = 15000
     SITEMAP_URL_SCHEME = 'http'
+    S3_ENDPOINT = os.getenv('S3_ENDPOINT')
 
 
 class TestingConfig(BaseConfig):
@@ -63,6 +64,7 @@ class TestingConfig(BaseConfig):
     NUM_OFFICERS = 120
     APPROVE_REGISTRATIONS = False
     SITEMAP_URL_SCHEME = 'http'
+    S3_ENDPOINT = os.getenv('S3_ENDPOINT')
 
 
 class ProductionConfig(BaseConfig):

OpenOversight/app/utils.py

@@ -6,6 +6,7 @@
 from io import BytesIO
 
 import boto3
+from botocore.client import Config
 from botocore.exceptions import ClientError
 import botocore
 import datetime
@@ -33,6 +34,13 @@
 SAVED_UMASK = os.umask(0o077)
 
 
+def get_s3_connection(config=None):
+    if current_app.config['S3_ENDPOINT']:
+        return boto3.client('s3', config=config, endpoint_url=current_app.config['S3_ENDPOINT'])
+    else:
+        return boto3.client('s3', config=config)
+
+
 def set_dynamic_default(form_field, value):
     # First we ensure no value is set already
     if not form_field.data:
@@ -226,8 +234,22 @@ def compute_hash(data_to_hash):
     return hashlib.sha256(data_to_hash).hexdigest()
 
 
+def get_presigned_image_url(s3_path: str):
+    unsigned = Config(signature_version=botocore.UNSIGNED)
+    s3_client_unsigned = get_s3_connection(unsigned)
+
+    url = s3_client_unsigned.generate_presigned_url(
+        'get_object',
+        Params={'Bucket': current_app.config['S3_BUCKET_NAME'], 'Key': s3_path}
+    )
+    return url
+
+
 def upload_obj_to_s3(file_obj, dest_filename):
-    s3_client = boto3.client('s3')
+    if current_app.config['S3_ENDPOINT']:
+        s3_client = get_s3_connection(Config(signature_version='s3v4'))
+    else:
+        s3_client = get_s3_connection()
 
     # Folder to store files in on S3 is first two chars of dest_filename
     s3_folder = dest_filename[0:2]
@@ -240,16 +262,7 @@ def upload_obj_to_s3(file_obj, dest_filename):
                              current_app.config['S3_BUCKET_NAME'],
                              s3_path,
                              ExtraArgs={'ContentType': s3_content_type, 'ACL': 'public-read'})
-
-    config = s3_client._client_config
-    config.signature_version = botocore.UNSIGNED
-    url = boto3.resource(
-        's3', config=config).meta.client.generate_presigned_url(
-        'get_object',
-        Params={'Bucket': current_app.config['S3_BUCKET_NAME'],
-                'Key': s3_path})
-
-    return url
+    return get_presigned_image_url(s3_path)
 
 
 def filter_by_form(form_data, officer_query, department_id=None):

OpenOversight/scripts/entrypoint.sh

@@ -2,6 +2,8 @@
 if [ "${DOCKER_BUILD_ENV:-}" == "production" ]; then
     flask run --host=0.0.0.0 --port=3000
 else
+    # forwarding port 9000 to minio for local S3 bucket
+    scripts/minio_forward.sh &
     yarn build
     yarn watch &
     flask run --host=0.0.0.0 --port=3000

OpenOversight/scripts/minio_forward.sh

@@ -0,0 +1 @@
+ncat -k -l localhost 9000 --sh-exec "ncat minio 9000"

OpenOversight/tests/conftest.py

@@ -16,7 +16,7 @@
 import os
 from PIL import Image as Pimage
 
-from OpenOversight.app import create_app, models
+from OpenOversight.app import create_app, models, utils
 from OpenOversight.app.utils import merge_dicts
 from OpenOversight.app.models import db as _db, Unit, Job, Officer
 from OpenOversight.tests.routes.route_helpers import ADMIN_EMAIL, ADMIN_PASSWORD
@@ -291,8 +291,13 @@ def add_mockdata(session):
     session.add_all(test_units)
     session.commit()
 
-    test_images = [models.Image(filepath='/static/images/test_cop{}.png'.format(x + 1), department_id=1) for x in range(5)] + \
-        [models.Image(filepath='/static/images/test_cop{}.png'.format(x + 1), department_id=2) for x in range(5)]
+    test_images = [
+        models.Image(
+            filepath=utils.get_presigned_image_url('/images/test_cop{}.png'.format(x + 1)),
+            department_id=1) for x in range(5)] + \
+        [models.Image(
+            filepath=utils.get_presigned_image_url('/images/test_cop{}.png'.format(x + 1)),
+            department_id=2) for x in range(5)]
 
     test_officer_links = [
         models.Link(
@@ -307,7 +312,7 @@ def add_mockdata(session):
             author='the internet'),
     ]
 
-    officers = [generate_officer() for o in range(NUM_OFFICERS)]
+    officers = [generate_officer() for _ in range(NUM_OFFICERS)]
     officers[0].links = test_officer_links
     session.add_all(officers)
     session.add_all(test_images)

docker-compose.yml

@@ -5,41 +5,60 @@ volumes:
     driver: local
 
 services:
- postgres:
-   restart: always
-   image: postgres:latest
-   environment:
-     POSTGRES_USER: openoversight
-     POSTGRES_PASSWORD: terriblepassword
-     POSTGRES_DB: openoversight-dev
-   volumes:
-     - postgres:/var/lib/postgresql/data
-   ports:
-     - "5432:5432"
+  postgres:
+    restart: always
+    image: postgres:latest
+    environment:
+      POSTGRES_USER: openoversight
+      POSTGRES_PASSWORD: terriblepassword
+      POSTGRES_DB: openoversight-dev
+    volumes:
+      - postgres:/var/lib/postgresql/data
+    ports:
+      - "5432:5432"
 
- web:
-   restart: always
-   build:
+  web:
+    restart: always
+    build:
       context: .
       args:
-      - DOCKER_BUILD_ENV
-      - TRAVIS_PYTHON_VERSION
+        - DOCKER_BUILD_ENV
+        - TRAVIS_PYTHON_VERSION
       dockerfile: ./dockerfiles/web/Dockerfile
-   environment:
-     AWS_ACCESS_KEY_ID: "${AWS_ACCESS_KEY_ID}"
-     AWS_SECRET_ACCESS_KEY: "${AWS_SECRET_ACCESS_KEY}"
-     AWS_DEFAULT_REGION: "${AWS_DEFAULT_REGION}"
-     S3_BUCKET_NAME: "${S3_BUCKET_NAME}"
-     APPROVE_REGISTRATIONS: "${APPROVE_REGISTRATIONS}"
-     FLASK_APP: app
-     FLASK_ENV: "${FLASK_ENV:-development}"
-   volumes:
-     - ./OpenOversight/:/usr/src/app/OpenOversight/:z
-   user: "${UID:?Docker-compose needs UID set to the current user id number. Try 'export UID' and run docker-compose again}"
-   links:
-     - postgres:postgres
-   expose:
-     - "3000"
-   command: scripts/entrypoint.sh
-   ports:
-     - "3000:3000"
+    environment:
+      AWS_ACCESS_KEY_ID: "${AWS_ACCESS_KEY_ID:-minio123}"
+      AWS_SECRET_ACCESS_KEY: "${AWS_SECRET_ACCESS_KEY:-minio123}"
+      S3_ENDPOINT: "${S3_ENDPOINT:-http://localhost:9000}"
+      AWS_DEFAULT_REGION: "${AWS_DEFAULT_REGION:-us-east-1}"
+      S3_BUCKET_NAME: "${S3_BUCKET_NAME:-openoversight-test}"
+      APPROVE_REGISTRATIONS: "${APPROVE_REGISTRATIONS}"
+      FLASK_APP: app
+      FLASK_ENV: "${FLASK_ENV:-development}"
+    volumes:
+      - ./OpenOversight/:/usr/src/app/OpenOversight/:z
+    user: "${UID:?Docker-compose needs UID set to the current user id number. Try 'export UID' and run docker-compose again}"
+    command: scripts/entrypoint.sh
+    ports:
+      - "3000:3000"
+
+  minio:
+    image: minio/minio
+    environment:
+      MINIO_ACCESS_KEY: "${AWS_ACCESS_KEY_ID:-minio123}"
+      MINIO_SECRET_KEY: "${AWS_SECRET_ACCESS_KEY:-minio123}"
+    command: server /data
+    volumes:
+      - ./OpenOversight/tests/resources/:/data:rw
+    ports:
+      - "9000:9000"
+
+  mc:
+    image: minio/mc
+    depends_on:
+      - minio
+    entrypoint: >
+      /bin/sh -c "
+      sleep 10;
+      /usr/bin/mc config host add minio http://minio:9000 ${AWS_ACCESS_KEY_ID:-minio123} ${AWS_SECRET_ACCESS_KEY:-minio123};
+      /usr/bin/mc policy set public minio/${S3_BUCKET_NAME:-openoversight-test};
+      "

dockerfiles/web/Dockerfile

@@ -11,7 +11,7 @@ ENV DISPLAY=:1
 
 # install apt dependencies
 RUN echo "deb http://deb.debian.org/debian stretch-backports main" > /etc/apt/sources.list.d/backports.list
-RUN apt-get update && apt-get install -y xvfb firefox-esr libpq-dev python3-dev && \
+RUN apt-get update && apt-get install -y xvfb firefox-esr libpq-dev python3-dev ncat && \
     apt-get install -y -t stretch-backports libsqlite3-0 && apt-get clean
 
 # install node

requirements.txt

@@ -2,6 +2,7 @@ Flask==1.0.2
 python-dotenv==0.7.1
 boto3==1.5.10
 werkzeug==1.0.1
+wtforms==2.3.3
 Flask-Bootstrap==3.3.7.1
 Flask-Limiter==1.0.1
 Flask-Login==0.4.1