diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 000000000..a97ef88e4 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,31 @@ +FROM ubuntu:18.04 + +RUN apt-get update -y +RUN apt-get upgrade -y + +RUN apt-get install -y build-essential wget +RUN apt-get install parallel -y + +ADD https://github.com/ufoscout/docker-compose-wait/releases/download/2.2.1/wait /wait +RUN chmod +x /wait + +RUN mkdir /colouring-london +COPY app /colouring-london/app + +ENV NODE_VERSION=16.13.2 +RUN apt-get install -y curl +RUN curl -o- https://raw.githubusercontent.com/nvm-sh/nvm/v0.39.0/install.sh | bash +ENV NVM_DIR=/root/.nvm +RUN . "$NVM_DIR/nvm.sh" && nvm install ${NODE_VERSION} +RUN . "$NVM_DIR/nvm.sh" && nvm use v${NODE_VERSION} +RUN . "$NVM_DIR/nvm.sh" && nvm alias default v${NODE_VERSION} +ENV PATH="/root/.nvm/versions/node/v${NODE_VERSION}/bin/:${PATH}" + +RUN npm install -g npm@latest + +WORKDIR ./colouring-london/app +RUN rm -rf node_modules +RUN npm install + +EXPOSE 8080 +CMD /wait && npm start \ No newline at end of file diff --git a/PostgresDockerfile b/PostgresDockerfile new file mode 100644 index 000000000..c9ec9b392 --- /dev/null +++ b/PostgresDockerfile @@ -0,0 +1,33 @@ +FROM ubuntu/postgres:12-20.04_beta + +RUN apt-get update -y +RUN apt-get upgrade -y + +RUN apt-get install -y postgresql-contrib libpq-dev +RUN apt-get install -y postgis +RUN apt-get install -y postgresql-12-postgis-3 +RUN apt-get install -y gdal-bin libspatialindex-dev libgeos-dev libproj-dev + +RUN apt-get install -y python3 python3-pip python3-dev + +RUN psql -d colouringlondon -U dockeruser -c "create extension postgis;" +RUN psql -d colouringlondon -U dockeruser -c "create extension pgcrypto;" +RUN psql -d colouringlondon -U dockeruser -c "create extension pg_trgm;" + +# RUN mkdir /colouring-london +# COPY app /colouring-london/app +# COPY migrations /colouring-london/migrations +# COPY etl /colouring-london/etl + +# RUN ls ./colouring-london/migrations/*.up.sql 2>/dev/null | while read -r migration; do psql -d colouringlondon < $migration; done; + +# RUN pip install --upgrade pip +# RUN pip install --upgrade setuptools wheel +# RUN pip install -r ./colouring-london/etl/requirements.txt + +# RUN python ./colouring-london/etl/get_test_polygons.py +# RUN ./colouring-london/etl/load_geometries_cl.sh ./ +# RUN psql -d colouringlondon -U dockeruser < ./colouring-london/app/migrations/002.index-geometries.up.sql +# RUN ./create_building_records_cl.sh +# RUN psql -d colouringlondon -U dockeruser < ./colouring-london/app/migrations/003.index-buildings.up.sql +# RUN ls ./colouring-london/migrations/*.up.sql 2>/dev/null | while read -r migration; do psql -d colouringlondon < $migration; done; \ No newline at end of file diff --git a/README.md b/README.md index 607734c51..0859709bd 100644 --- a/README.md +++ b/README.md @@ -26,6 +26,11 @@ for download under a liberal open data license ## Setup and run +#### Quick run the application + +- `chmod +x init-user-db.sh` +- `docker compose up` + #### Test the application: You can try out the Colouring London application by setting up your own development environment, which includes the option to load test data from OpenStreetMaps (OSM). See [docs/setup-dev-environment](docs/setup-dev-environment.md). diff --git a/docker-compose.yml b/docker-compose.yml new file mode 100644 index 000000000..c891035a3 --- /dev/null +++ b/docker-compose.yml @@ -0,0 +1,36 @@ +version: "3" +services: + colouring-london: + container_name: colouring-london + build: + context: . + dockerfile: Dockerfile + depends_on: + - db + environment: + DATABASE_URL: postgres://dockeruser:postgres@postgres-colouring-london:5432/colouringlondon + NODE_ENV: development + PORT: 8080 + APP_COOKIE_SECRET: 123456 + ports: + - "8080:8080" + db: + container_name: postgres-colouring-london + # build: + # context: . + # dockerfile: PostgresDockerfile + image: ubuntu/postgres:12-20.04_beta + # image: postgis/postgis:12-3.2-alpine + ports: + - "5432:5432" + volumes: + - ./postgresdata:/var/lib/postgresql/data + # - ./migrations:/colouring-london/migrations + # - ./etl:/colouring-london/etl + - ./init-user-db.sh:/docker-entrypoint-initdb.d/init-user-db.sh + # - ./migrations/001.core.up.sql:/docker-entrypoint-initdb.d/001.core.up.sql + restart: always + environment: + POSTGRES_USER: dockeruser + POSTGRES_PASSWORD: postgres + POSTGRES_DB: colouringlondon \ No newline at end of file diff --git a/etl/create_building_records_cl.sh b/etl/create_building_records_cl.sh new file mode 100644 index 000000000..57464bb2b --- /dev/null +++ b/etl/create_building_records_cl.sh @@ -0,0 +1,9 @@ +#!/usr/bin/env bash + +# +# Create corresponding 'building' record with +# id: , +# doc: {}, +# geom_id: +# +psql -d colouringlondon -c "INSERT INTO buildings ( geometry_id, ref_toid ) SELECT geometry_id, source_id from geometries;" diff --git a/etl/load_geometries_cl.sh b/etl/load_geometries_cl.sh new file mode 100644 index 000000000..81f73ac27 --- /dev/null +++ b/etl/load_geometries_cl.sh @@ -0,0 +1,32 @@ +#!/usr/bin/env bash + +# +# Load geometries from GeoJSON to Postgres +# - assume postgres connection details are set in the environment using PGUSER, PGHOST etc. +# +: ${1?"Usage: $0 ./path/to/mastermap/dir"} + +mastermap_dir=$1 + +# +# Create 'geometry' record with +# id: , +# source_id: , +# geom: +# +find $mastermap_dir -type f -name '*.3857.csv' \ +-printf "$mastermap_dir/%f\n" | \ +parallel \ +cat {} '|' psql -d colouringlondon -c "\"COPY geometries ( geometry_geom, source_id ) FROM stdin WITH CSV HEADER;\"" + +# +# Delete any duplicated geometries (by TOID) +# +psql -d colouringlondon -c "DELETE FROM geometries a USING ( + SELECT MIN(ctid) as ctid, source_id + FROM geometries + GROUP BY source_id + HAVING COUNT(*) > 1 +) b +WHERE a.source_id = b.source_id +AND a.ctid <> b.ctid;" diff --git a/init-user-db.sh b/init-user-db.sh new file mode 100755 index 000000000..3c3d626a3 --- /dev/null +++ b/init-user-db.sh @@ -0,0 +1,27 @@ +#!/bin/bash +apt-get update -y +apt-get upgrade -y + +apt-get install -y postgresql-contrib libpq-dev postgis +apt-get install -y postgresql-12-postgis-3 +apt-get install -y gdal-bin libspatialindex-dev libgeos-dev libproj-dev + +apt-get install -y python3 python3-pip python3-dev + +# Do I need to create a db? Look at the dockerhub +# psql -d colouringlondon -U dockeruser -c "SELECT 1 FROM pg_user WHERE usename = 'dockeruser';" | grep -q 1 || psql -d colouringlondon -U dockeruser -c "CREATE ROLE dockeruser SUPERUSER LOGIN PASSWORD 'postgres';" +# psql -d colouringlondon -U dockeruser -c "SELECT 1 FROM pg_database WHERE datname = 'colouringlondon';" | grep -q 1 || -u postgres createdb -E UTF8 -T template0 --locale=en_US.utf8 -O dockeruser colouringlondon + +psql -c "create extension postgis;" +psql -c "create extension pgcrypto;" +psql -c "create extension pg_trgm;" + +ls ./colouring-london/migrations/*.up.sql 2>/dev/null | while read -r migration; do psql < $migration; done; + +pip install --upgrade pip +pip install --upgrade setuptools wheel +pip install -r ./colouring-london/etl/requirements.txt + +python ./colouring-london/etl/get_test_polygons.py +./colouring-london/etl/load_geometries_cl.sh ./ +./colouring-london/etl/create_building_records_cl.sh \ No newline at end of file