diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index f077db93..175622d4 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -215,7 +215,7 @@ jobs: - name: start database, server, and app run: | echo "==> start database and data containers" - docker-compose -f "${{ env.DOCKER_COMPOSE_PATH }}" up -d + docker compose -f "${{ env.DOCKER_COMPOSE_PATH }}" up -d echo "==> wait for data container to finish seeding database container" DATA_CONTAINER=$(docker compose -f "${{ env.DOCKER_COMPOSE_PATH }}" ps -a --format '{{.Name}}' mongo-seed) @@ -237,7 +237,7 @@ jobs: pid=$(lsof -i :8080 -t) && kill ${pid} echo "==> stop database" - docker-compose -f "${{ env.DOCKER_COMPOSE_PATH }}" down + docker compose -f "${{ env.DOCKER_COMPOSE_PATH }}" down - uses: actions/upload-artifact@v4 if: ${{ !cancelled() }} diff --git a/README.md b/README.md index 1ac18756..878ae3ce 100644 --- a/README.md +++ b/README.md @@ -11,12 +11,13 @@ What you need to run this app: - `node` and `npm` (`brew install node`) - Ensure you're running the latest versions Node `v16.x.x`+ and NPM `8.x.x`+ -- A version 6.0+ [MongoDB](https://www.mongodb.com/docs/manual/administration/install-community/) instance running on your local machine - -- You can optionally use a GUI like [Compass](https://www.mongodb.com/docs/compass/current/) or [Studio3T](https://studio3t.com/knowledge-base/articles/installation/) with your local database - +- If running against a local database: + - A version 6.0+ [MongoDB](https://www.mongodb.com/docs/manual/administration/install-community/) instance running on your local machine + - You can optionally use a GUI like [Compass](https://www.mongodb.com/docs/compass/current/) or [Studio3T](https://studio3t.com/knowledge-base/articles/installation/) with your local database - Note that only Studio3T is compatible with the [AWS DocumentDB](https://docs.aws.amazon.com/documentdb/latest/developerguide/what-is.html) instances in Agora's dev, stage and prod environments. Either GUI tool will work with your local Mongo instance. - +- If running against a conatinerized database: + - [Docker Desktop](https://docs.docker.com/desktop/) + - [jq](https://jqlang.github.io/jq/) (`brew install jq`) > If you have `nvm` installed, which is highly recommended (`brew install nvm`) you can do a `nvm install --lts && nvm use` in `$` to run with the latest Node LTS. You can also have this `zsh` done for you [automatically](https://github.com/creationix/nvm#calling-nvm-use-automatically-in-a-directory-with-a-nvmrc-file) @@ -170,9 +171,9 @@ Go to [http://localhost:8080](http://localhost:8080) ### 6 - Use containerized database -1. Install Docker, if necessary. +1. Install [Docker Desktop](https://docs.docker.com/desktop/) and [jq](https://jqlang.github.io/jq/) (`brew install jq`), if necessary. 2. Update `data-file` and `data-version` in `package.json` to reflect the desired data release version, if necessary. -3. Create an environment file: `npm run create-env`. +3. Create an environment file: `npm run create-env`. The credentials for connecting to the containerized MongoDB will be included in the generated `.env` file, where the connection string will be: `mongodb://{DB_USER}:{DB_PASS}@localhost:{DB_PORT}`. 4. Start the containerized database: `npm run docker:db:start`. The necessary images will be pulled from GHCR. If you would like to use a different image, update DATA_IMAGE_PATH in .env. If the desired image does not exist, see steps below to create the desired image. 5. Run the server and app against the containerized database: `npm run docker:dev`. 6. Stop the containerized database: `npm run docker:db:stop`. @@ -185,14 +186,15 @@ The `sage-bionetworks` package will contain images for data releases that have b If a dev needs to create an image for a data release that does not yet exist in the Sage-Bionetworks package, they should follow these steps: -1. Create a new branch. -2. Update the `package.json` to reflect the appropriate `data-file` and `data-version` files. -3. If necessary, update `./scripts/collections.csv` to specify new collections and `./scripts/mongo-create-Indexes.js` to specify new indexes. -4. Commit the changes. -5. Push the changes to your remote fork to trigger a run of the `ci.yml` workflow. -6. The new image will be available in your user namespaced GHCR package, e.g. `https://github.com/hallieswan/Agora/pkgs/container/agora-data-nonmonorepo`. -7. Update your local `.env` file so `DATA_IMAGE_PATH` points to the newly created image, e.g. `ghcr.io/hallieswan/agora-data-nonmonorepo:syn13363290.68`. -8. Start the containerized database: `npm run docker:db:start`. +1. Set up GitHub secrets/variables in your remote fork as described [here](docker/README.md#workflow-setup), if necessary. +2. Create a new branch. +3. Update the `package.json` to reflect the appropriate `data-file` and `data-version` files. +4. If necessary, update `./scripts/collections.csv` to specify new collections and `./scripts/mongo-create-Indexes.js` to specify new indexes. +5. Commit the changes. +6. Push the changes to your remote fork to trigger a run of the `ci.yml` workflow. +7. The new image will be available in your user namespaced GHCR package, e.g. `https://github.com/hallieswan/Agora/pkgs/container/agora-data-nonmonorepo`. +8. Update your local `.env` file so `DATA_IMAGE_PATH` points to the newly created image, e.g. `ghcr.io/hallieswan/agora-data-nonmonorepo:syn13363290.68`. +9. Start the containerized database: `npm run docker:db:start`. # Development diff --git a/docker/README.md b/docker/README.md index 78416d8b..82aca5e9 100644 --- a/docker/README.md +++ b/docker/README.md @@ -7,15 +7,15 @@ Files in this directory: ## Workflow Setup -The following secrets and variables need to be set up in GitHub for the `ci.yml` workflow to create the `agora-data-nonmonorepo` Docker image: +The following secrets and variables need to be set up in GitHub for the `ci.yml` workflow to create the `agora-data-nonmonorepo` Docker image. See instructions for creating a GitHub environment and adding secrets/variables [here](https://docs.github.com/en/actions/deployment/targeting-different-environments/using-environments-for-deployment#creating-an-environment). e2e Environment secrets: -| Variable | Description | Example | -| ------------------ | ------------------------------------------ | ----------------- | -| DB_USER | The database user | dbuser | -| DB_PASS | The database password | supersecret | -| SYNAPSE_AUTH_TOKEN | The Synapse service user view/download PAT | token-string-here | +| Secret | Description | Example | +| ------------------ | ------------------------------------------------------ | ----------------- | +| DB_USER | The database user | dbuser | +| DB_PASS | The database password | supersecret | +| SYNAPSE_AUTH_TOKEN | View/download PAT for `synapse-service-agora-e2e` user | token-string-here | e2e Environment variables: diff --git a/package.json b/package.json index bee07c33..50e50768 100644 --- a/package.json +++ b/package.json @@ -41,11 +41,11 @@ "e2e": "playwright test --trace on", "e2e:ui": "playwright test --ui", "e2e:update": "npm install -D @playwright/test@latest; npx playwright install --with-deps; npx playwright --version", - "create-env": "DB_PASS=$(openssl rand -base64 12) DATA_IMAGE_TAG=$(npm run docker:data-image-tag -s) envsubst < .env.example > .env", + "create-env": "[ ! -f .env ] && DB_PASS=$(openssl rand -base64 12) DATA_IMAGE_TAG=$(npm run docker:data-image-tag -s) envsubst < .env.example > .env", "docker:data-image-tag": "jq -r '.\"data-file\" + \".\" + .\"data-version\"' package.json", "docker:db:seed-complete": "echo 'Waiting for docker db to finish loading data (~30s)....'; DATA_CONTAINER=$(docker compose --env-file .env -f ./docker/docker-compose.yml ps -a --format '{{.Name}}' mongo-seed); docker wait ${DATA_CONTAINER}", - "docker:db:start": "docker-compose --env-file .env -f ./docker/docker-compose.yml up -d; npm run docker:db:seed-complete", - "docker:db:stop": "docker-compose --env-file .env -f ./docker/docker-compose.yml down", + "docker:db:start": "docker compose --env-file .env -f ./docker/docker-compose.yml up -d; npm run docker:db:seed-complete", + "docker:db:stop": "docker compose --env-file .env -f ./docker/docker-compose.yml down", "docker:dev": "set -a; source .env; set +a; MONGODB_HOST=\"localhost\" MONGODB_PORT=\"${DB_PORT}\" APP_ENV=\"e2e\" npm run dev" }, "pre-commit": [