bacpop-25 use gps v6 db

.github/workflows/jestCI.yml

@@ -50,7 +50,7 @@ jobs:
       with:
         node-version: ${{ matrix.node-version }}
         cache: 'npm'
-        cache-dependency-path: ./app/client/package-lock.json
+        cache-dependency-path: ./app/server/package-lock.json
     - run: npm ci
     - name: get config file
       run: |

README.md

@@ -1,5 +1,36 @@
 # beebop
 
+## Docker Quick Start
+
+Run the dockerised app along with proxy and all dependencies:
+
+```
+    ./scripts/run_docker_decrypt
+```
+
+You may need to update your version of Docker and Docker Compose: see [here](https://docs.docker.com/engine/install/ubuntu/) for instructions on updating on Ubuntu. 
+
+By default this will configure the nginx proxy for host localhost. To deploy with a different hostname, pass it as an argument, e.g.
+```
+./scripts/run_docker_decrypt beebop.dide.ic.ac.uk
+```
+
+This will also populate app config with secrets from the vault. If you are not running the script for the first time,
+or not for the first time since running the app outside docker, you can omit this step by running the `run_docker` script.
+
+Bring down the app with
+```
+    ./scripts/stop_docker
+```
+
+Docker images are built on CI using `./proxy/docker/build`, `./app/server/docker/build`. If you want
+to generate them from changed local sources you can run those same scripts locally to build images.
+
+To target a branch of `beebop_py`, set `API_BRANCH` in `scripts/common`.
+
+When running locally in docker, the backend is serving from `beebop_beebop-server_1`, and the front end from the proxy
+container `beebop_proxy_1`.
+
 ## Local development
 
 Clone the repository to your computer with
@@ -15,7 +46,8 @@ docker --version
 ```
 
 
-If you run the application for the first time, you need to replace the secrets in the config file in `app/server/src/resources` first. 
+If you run the application for the first time (or for the first time after running in docker), you need to replace the 
+secrets in the config file in `app/server/src/resources` first. 
 Login to the vault:
 ```
 export VAULT_ADDR=https://vault.dide.ic.ac.uk:8200

app/client/tests/unit/components/SavedProjects.spec.ts

@@ -2,7 +2,7 @@
 import EditProjectName from "@/components/projects/EditProjectName.vue";
 
 const { toLocaleString } = Date.prototype;
-// eslint-disable-next-line no-extend-native
+// eslint-disable-next-line no-extend-native, func-names
 Date.prototype.toLocaleString = function (locale: any = undefined, ...args: any) {
     const options = args[0];
     return toLocaleString.call(this, "en-GB", { ...options, timeZone: "UTC" });

app/server/docker/Dockerfile

@@ -8,4 +8,5 @@ RUN npm ci
 COPY . /app
 EXPOSE 4000
 RUN npm run build
+RUN npm install -g ts-node
 ENTRYPOINT ["docker/entrypoint.sh"]

app/server/docker/entrypoint.sh

@@ -7,4 +7,4 @@ while [ ! -e $PATH_CONFIG ]; do
   sleep 1
 done
 
-node dist/index.js --config /app/src/resources
+ts-node --transpile-only src/index.ts --config src/resources

app/server/jest.config.js

@@ -2,5 +2,6 @@
 module.exports = {
   preset: 'ts-jest',
   testEnvironment: 'node',
-  testTimeout: 120000
+  testTimeout: 120000,
+  maxWorkers: 1
 };

app/server/src/resources/config.json.in.development

@@ -8,6 +8,5 @@
     "GOOGLE_CLIENT_SECRET": "$GOOGLE_SECRET",
     "SESSION_SECRET": "$EXPRESS_SESSION_SECRET",
     "GITHUB_CLIENT_ID":"$GITHUB_ID",
-    "GITHUB_CLIENT_SECRET":"$GITHUB_SECRET",
-    "skip_auth": false
+    "GITHUB_CLIENT_SECRET":"$GITHUB_SECRET"
 }

app/server/src/resources/config.json.in.docker

@@ -1,13 +1,12 @@
 {
     "server_port": 4000,
     "api_url": "http://beebop-py-api:5000",
-    "client_url": "http://localhost",
+    "client_url": "https://localhost",
     "server_url": "https://localhost/api",
     "redis_url": "redis://beebop-redis:6379",
-    "GOOGLE_CLIENT_ID": "1234",
-    "GOOGLE_CLIENT_SECRET": "1234",
+    "GOOGLE_CLIENT_ID": "$DOCKER_GOOGLE_ID",
+    "GOOGLE_CLIENT_SECRET": "$DOCKER_GOOGLE_SECRET",
     "SESSION_SECRET": "1234",
-    "GITHUB_CLIENT_ID": "1234",
-    "GITHUB_CLIENT_SECRET": "1234",
-    "skip_auth": true
+    "GITHUB_CLIENT_ID": "$DOCKER_GITHUB_ID",
+    "GITHUB_CLIENT_SECRET": "$DOCKER_GITHUB_SECRET"
 }

app/server/src/resources/config.json.in.production

@@ -8,6 +8,5 @@
     "GOOGLE_CLIENT_SECRET": "$PROD_GOOGLE_SECRET",
     "SESSION_SECRET": "$PROD_EXPRESS_SESSION_SECRET",
     "GITHUB_CLIENT_ID":"$PROD_GITHUB_ID",
-    "GITHUB_CLIENT_SECRET":"$PROD_GITHUB_SECRET",
-    "skip_auth": false
+    "GITHUB_CLIENT_SECRET":"$PROD_GITHUB_SECRET"
 }

app/server/tests/integration/errors.test.ts

@@ -1,5 +1,6 @@
 import {flushRedis, get, post, saveRedisHash, saveRedisSet} from "./utils";
 import {uid} from "uid";
+import {setTimeout} from "timers/promises";
 import {testSample} from "./testSample";
 
 describe("Error handling", () => {
@@ -11,8 +12,9 @@ describe("Error handling", () => {
         connectionCookie = response.headers["set-cookie"][0];
     });
 
-    const sampleHash = Object.keys(testSample.names)[0];
-    const sampleFileName = testSample.names[sampleHash] ;
+    const names = testSample("" ,"").names;
+    const sampleHash = Object.keys(names)[0];
+    const sampleFileName = names[sampleHash] ;
     const sampleId = `${sampleHash}:${sampleFileName}`;
 
     const newProject = async () => {
@@ -40,22 +42,23 @@ describe("Error handling", () => {
         expect(amrRes.status).toBe(200);
 
         // 3. Run poppunk and wait til it finishes
-        testSample.projectId = projectId;
-        const poppunkRes = await post(`poppunk`, testSample, connectionCookie);
+        const fakeProjectHash = `${projectId}ABC`;
+        const projectData = testSample(fakeProjectHash, projectId);
+        const poppunkRes = await post(`poppunk`, projectData, connectionCookie);
         expect(poppunkRes.status).toBe(200);
-        const counter = 0;
+        expect(poppunkRes.data.data.assign).not.toBe("")
+        let counter = 0;
         let finished = false;
         while (!finished && counter < 100) {
-            await new Promise(resolve => {
-                setTimeout(() => {resolve(""), 1000})
-            });
-            const statusRes = await post("status", {hash: testSample.projectHash}, connectionCookie);
+            await setTimeout(2000);
+            const statusRes = await post("status", {hash: fakeProjectHash}, connectionCookie);
             expect(statusRes.status).toBe(200);
             const statusValues = statusRes.data.data;
-            if (statusValues.assign === "finished" && statusValues.microreact === "finished" && statusValues.network === "finished") {
+            if (statusValues && statusValues.assign === "finished" && statusValues.microreact === "finished" && statusValues.network === "finished") {
                 finished = true;
                 break;
             }
+            counter = counter + 1;
         }
         expect(finished).toBe(true);
         return projectId;

docker-compose.yml

@@ -15,19 +15,19 @@ services:
       - beebop-py-api
       - beebop-redis
   beebop-py-api:
-    image: mrcide/beebop-py:main
+    image: mrcide/beebop-py:${API_BRANCH}
     depends_on:
       - beebop-redis
     volumes:
       - beebop-storage:/beebop/storage
     environment:
       - REDIS_HOST=beebop-redis
       - STORAGE_LOCATION=./storage
-      - DB_LOCATION=./storage/GPS_v4_references
+      - DB_LOCATION=${DB_LOCATION}
   beebop-redis:
     image: redis:5.0
   beebop-py-worker:
-    image: mrcide/beebop-py:main
+    image: mrcide/beebop-py:${API_BRANCH}
     depends_on:
       - beebop-redis
     command:

scripts/common

@@ -0,0 +1,2 @@
+export API_BRANCH=main
+export DB_LOCATION="./storage/GPS_v6_references"

scripts/decrypt_config

@@ -11,6 +11,10 @@ FILE_CLEAR_SERVER=app/server/src/resources/config.json
 
 export GITHUB_ID=$(vault read -field=clientid secret/beebop/auth/github)
 export GITHUB_SECRET=$(vault read -field=secret secret/beebop/auth/github)
+export DOCKER_GITHUB_ID=$(vault read -field=clientid secret/beebop/auth/devdocker/github)
+export DOCKER_GITHUB_SECRET=$(vault read -field=secret secret/beebop/auth/devdocker/github)
+export DOCKER_GOOGLE_ID=$(vault read -field=clientid secret/beebop/auth/devdocker/google)
+export DOCKER_GOOGLE_SECRET=$(vault read -field=secret secret/beebop/auth/devdocker/google)
 export PROD_GITHUB_ID=$(vault read -field=clientid secret/beebop/auth/production/github)
 export PROD_GITHUB_SECRET=$(vault read -field=secret secret/beebop/auth/production/github)
 export PROD_GOOGLE_ID=$(vault read -field=clientid secret/beebop/auth/production/google)

scripts/docker_smoke_test

@@ -8,6 +8,7 @@ function cleanup() {
 }
 trap cleanup EXIT
 
+cp app/server/src/resources/config.json.in.docker app/server/src/resources/config.json
 $HERE/run_docker
 
 URL=https://localhost

scripts/run_client

@@ -1 +1,5 @@
+#!/usr/bin/env bash
+set -ex
+
+npm ci --prefix app/client
 npm run --prefix app/client serve

scripts/run_dependencies

@@ -1,8 +1,13 @@
+#!/usr/bin/env bash
+set -ex
+
+HERE=$(realpath "$(dirname $0)")
+. $HERE/common
+
 NETWORK=beebop_nw
 VOLUME=beebop-storage
 NAME_REDIS=beebop-redis
 NAME_API=beebop-py-api
-API_BRANCH=main
 NAME_WORKER=beebop-py-worker
 PORT=5000
 
@@ -14,13 +19,13 @@ docker network create $NETWORK > /dev/null || /bin/true
 docker run -d --rm --name $NAME_REDIS --network=$NETWORK -p 6379:6379 redis:5.0
 docker run -d --rm --name $NAME_WORKER --network=$NETWORK \
        --env=REDIS_HOST="$NAME_REDIS" \
+       --pull always \
        -v $VOLUME:/beebop/storage \
        mrcide/beebop-py:$API_BRANCH rqworker
 docker run -d --rm --name $NAME_API --network=$NETWORK \
-       --pull always \
        --env=REDIS_HOST="$NAME_REDIS" \
        --env=STORAGE_LOCATION="./storage" \
-       --env=DB_LOCATION="./storage/GPS_v4_references" \
+       --env=DB_LOCATION="$DB_LOCATION" \
        -v $VOLUME:/beebop/storage \
        -p $PORT:5000 \
        mrcide/beebop-py:$API_BRANCH

scripts/run_docker

@@ -1,7 +1,8 @@
 #!/usr/bin/env bash
-set -e
+set -ex
 
-export GIT_SHA=$(git -C . rev-parse --short=7 HEAD)
+HERE=$(realpath "$(dirname $0)")
+. $HERE/common
 
 if [ "$#" -eq 1 ]; then
   HOST=$1
@@ -11,10 +12,13 @@ else
   SSL_PATH=ssl
 fi
 export HOST
-docker-compose up -d
-docker cp app/server/src/resources/config.json.in.docker beebop_beebop-server_1:/app/src/resources/config.json
-docker cp proxy/ssl/dhparam.pem beebop_proxy_1:/run/proxy/
-docker cp proxy/$SSL_PATH/certificate.pem beebop_proxy_1:/run/proxy/
-docker cp proxy/$SSL_PATH/key.pem beebop_proxy_1:/run/proxy/
+
+export GIT_SHA=$(git -C . rev-parse --short=7 HEAD)
+
+docker compose up -d --pull always
+docker cp app/server/src/resources/config.json beebop-beebop-server-1:/app/src/resources/config.json
+docker cp proxy/ssl/dhparam.pem beebop-proxy-1:/run/proxy/
+docker cp proxy/$SSL_PATH/certificate.pem beebop-proxy-1:/run/proxy/
+docker cp proxy/$SSL_PATH/key.pem beebop-proxy-1:/run/proxy/
 docker run --rm -v beebop_beebop-storage:/beebop/storage mrcide/beebop-py:main \
        ./scripts/download_db --small storage

scripts/run_docker_decrypt

@@ -0,0 +1,6 @@
+HERE=$(realpath "$(dirname $0)")
+
+export VAULT_ADDR=https://vault.dide.ic.ac.uk:8200
+vault login -method=github
+. $HERE/decrypt_config docker
+. $HERE/run_docker

scripts/run_server

@@ -1 +1,5 @@
+#!/usr/bin/env bash
+set -ex
+
+npm ci --prefix app/server
 BEEBOP_TEST="true" npm run --prefix app/server express

scripts/stop_docker

@@ -1,4 +1,5 @@
 #!/usr/bin/env bash
 set -e
 
-docker-compose down
+docker compose down
+docker volume rm beebop_beebop-storage