feat: added s3 to atlas-server (#120)

This PR adds an S3 cache for the tiles object, and redirects to it when
present.

You can test this branch locally is you have Docker, by running
`docker-compose up` (it will start MinIO, a local s3).
Then add these to your `.env`:
```
AWS_ACCESS_KEY_ID=admin
AWS_SECRET_ACCESS_KEY=password
AWS_S3_BUCKET=atlas-server
AWS_S3_REGION=us-east-1
AWS_S3_ENDPOINT=http://0.0.0.0:9000
```
Finally, build and run the server: `npm run build && npm start`.

You should see the tiles json should be stored in your local minio
(there's a browser at `http://localhost:9001/`) and the `/v1/tiles` and
`/v2/tiles` enpoint should redirect there.

.env.defaults

@@ -4,13 +4,18 @@ HTTP_SERVER_PORT=5000
 HTTP_SERVER_HOST=0.0.0.0
 CORS_ORIGIN=*
 CORS_METHOD=*
-SUBGRAPH_URL=https://api.thegraph.com/subgraphs/name/decentraland/marketplace
+SUBGRAPH_URL=https://subgraph.decentraland.org/marketplace
 SUBGRAPH_COMPONENT_QUERY_TIMEOUT=30000
 API_BATCH_SIZE=1000
 API_CONCURRENCY=5
 REFRESH_INTERVAL=60
 IMAGE_BASE_URL=https://api.decentraland.org/v2
-EXTERNAL_BASE_URL=https://market.decentraland.org
+EXTERNAL_BASE_URL=https://decentraland.org/marketplace
 LAND_CONTRACT_ADDRESS=0xf87e31492faf9a91b02ee0deaad50d51d56d5d4d
 ESTATE_CONTRACT_ADDRESS=0x959e104e1a4db6317fa58f8295f586e1a978c297
 SIGNATURES_SERVER_URL=https://signatures-api.decentraland.org
+AWS_ACCESS_KEY_ID=admin
+AWS_SECRET_ACCESS_KEY=password
+AWS_S3_BUCKET=atlas-server
+AWS_S3_REGION=us-east-1
+AWS_S3_ENDPOINT=http://0.0.0.0:9000

.gitignore

@@ -31,4 +31,5 @@ src/data
 tests/**/*.spec.js
 coverage
 dist
-.env
+.env
+data

README.md

@@ -10,6 +10,11 @@
 2. Run `npm run build`
 3. Run `npm start`
 
+## Local development
+
+1. Run `docker-compose up` to start the minio service
+2. Run `npm start` to start the server
+
 ## Config
 
 The env variables and their default values are the following:
@@ -22,6 +27,13 @@ SUBGRAPH_COMPONENT_QUERY_TIMEOUT=30000
 API_BATCH_SIZE=1000
 API_CONCURRENCY=10
 REFRESH_INTERVAL=60
+
+# AWS S3 Configuration for local development (using minio)
+AWS_ACCESS_KEY_ID=admin
+AWS_SECRET_ACCESS_KEY=password
+AWS_S3_BUCKET=atlas-server
+AWS_S3_REGION=us-east-1
+AWS_S3_ENDPOINT=http://0.0.0.0:9000
 ```
 
 You can `cp .env.example .env` and tweak the ones you want to change

docker-compose.yml

@@ -0,0 +1,38 @@
+# Utilitarian compose file to locally run required external services
+# required by the asset-packs. Just by running `docker-compose up`
+# you will have a http-server and an s3 compatible local object storage
+# ready to be used.
+
+version: '3.8'
+
+services:
+  # Object storage compatible with aws-sdk.
+  # Comes with a UI that can be accessed via http://localhost:9001.
+  # https://min.io/
+  minio:
+    image: minio/minio
+    environment:
+      - MINIO_ROOT_USER=admin
+      - MINIO_ROOT_PASSWORD=password
+    ports:
+      - 9000:9000
+      - 9001:9001
+    volumes:
+      - minio_data:/data
+    command: server /data --console-address ":9001"
+
+  # Companion for the minio service in charge of initialization and
+  # provides an mc client for operating directly with the storage
+  # https://docs.min.io/minio/baremetal/reference/minio-cli/minio-mc.html
+  minio-mc:
+    image: minio/mc
+    depends_on:
+      - minio
+    env_file:
+      - .env
+    volumes:
+      - ./minio-mc-entrypoint.sh:/scripts/entrypoint.sh
+    entrypoint: /scripts/entrypoint.sh
+
+volumes:
+  minio_data:

minio-mc-entrypoint.sh

@@ -0,0 +1,21 @@
+#!/bin/bash
+
+# Required as the minio service on the docker-compose file might
+# not be completely up on the execution of this file.
+sleep 5
+
+# Create an alias to the minio service so it can be accessed easily
+mc alias set minio http://minio:9000 admin password
+
+# Check if the bucket already exists
+if mc find minio/$AWS_S3_BUCKET ; then
+  echo "Bucket \"$AWS_S3_BUCKET\" already exists, no need to create it again"
+else
+  # Create the bucket and set the policy to public so anything can 
+  # be downloaded or uploaded
+  mc mb minio/$AWS_S3_BUCKET
+  mc anonymous set public minio/$AWS_S3_BUCKET
+fi
+
+# Keep the service running so it can be accessed later with docker-compose exec
+tail -f /dev/null