Merge pull request #121 from Sage-Bionetworks/staging

AG-1339: mv68 to prod

.github/CODEOWNERS

@@ -0,0 +1 @@
+* @Sage-Bionetworks/sagebio-it @Sage-Bionetworks/Agora-Admin

.github/workflows/main.yaml

@@ -0,0 +1,36 @@
+name: main
+
+on:
+  pull_request:
+    branches: ['*']
+  push:
+    branches: ['develop', 'staging', 'prod' ]
+
+jobs:
+  tests:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - uses: pre-commit/[email protected]
+
+  deploy:
+    if: ${{ github.event_name == 'push' }}
+    needs: ["tests"]
+    # self hosted runner labels are setup in github to match branch names
+    runs-on: [self-hosted, "${{ github.ref_name }}"]
+    # variables in context environments are setup in github to match branch names
+    environment:
+      name: ${{ github.ref_name }}
+
+    steps:
+      # use older checkout version due to https://github.com/dawidd6/action-download-artifact/issues/261
+      - uses: actions/checkout@v2
+      - name: Import Synapse Data
+        run: ./import-data.sh $BRANCH $SYNAPSE_USERNAME $SYNAPSE_PASSWORD $DB_HOST $DB_USER $DB_PASS
+        env:
+          BRANCH: ${{ github.ref_name }}
+          SYNAPSE_USERNAME: ${{ secrets.SYNAPSE_USERNAME }}
+          SYNAPSE_PASSWORD: ${{ secrets.SYNAPSE_PASSWORD }}
+          DB_HOST: ${{ secrets.DB_HOST }}
+          DB_USER: ${{ secrets.DB_USER }}
+          DB_PASS: ${{ secrets.DB_PASS }}

.pre-commit-config.yaml

@@ -0,0 +1,19 @@
+repos:
+    - repo: https://github.com/pre-commit/pre-commit-hooks
+      rev: v4.5.0
+      hooks:
+          - id: end-of-file-fixer
+          - id: trailing-whitespace
+    - repo: https://github.com/adrienverge/yamllint
+      rev: v1.33.0
+      hooks:
+          - id: yamllint
+    - repo: https://github.com/Lucas-C/pre-commit-hooks
+      rev: v1.5.4
+      hooks:
+          - id: remove-tabs
+    - repo: https://github.com/sirosen/check-jsonschema
+      rev: 0.27.0
+      hooks:
+          - id: check-github-workflows
+          - id: check-github-actions

.yamllint

@@ -0,0 +1,29 @@
+---
+
+extends: default
+
+rules:
+  braces:
+    level: warning
+    max-spaces-inside: 1
+  brackets:
+    level: warning
+    max-spaces-inside: 1
+  colons:
+    level: warning
+  commas:
+    level: warning
+  comments: disable
+  comments-indentation: disable
+  document-start: disable
+  empty-lines:
+    level: warning
+  hyphens:
+    level: warning
+  indentation:
+    level: warning
+    indent-sequences: consistent
+  line-length: disable
+  truthy: disable
+  new-line-at-end-of-file:
+    level: warning

README.md

@@ -1,48 +1,74 @@
 # Overview
-Agora Data Manager is a tool that loads the JSON files into Agora's document database instances in our AWS environments.
+Agora Data Manager is a tool that loads the JSON files into Agora's document database
+instances in our AWS environments.
 
 # Purpose
 This project allows Agora maintainers to update the Agora database with
 new versions of gene data from Synapse.  This is a manually triggered,
-self-service update. 
+self-service update.
 
 # Execution
 
 ![alt text][db_update]
 
-# Worflow
+# Workflow
 
 To deploy an updated data version to the Agora development database
 1. Increment `data-version` in `data-manifest.json` on the `develop` branch.
 2. Commit the change
-3. The [CI system](https://travis-ci.org/Sage-Bionetworks/agora-data-manager) automatically updates the dev DB
+3. The Github action CI system automatically updates the dev DB
 
 
 To deploy an updated data version to the Agora staging database:
 1. Merge the data-version update from the dev branch to the staging branch.
-2. The [CI system](https://travis-ci.org/Sage-Bionetworks/agora-data-manager) automatically updates the staging DB
+2. The Github action CI system automatically updates the dev DB
 
 To deploy an updated data version to the Agora production database:
 1. Merge the data-version update from the staging branch to the production branch.
-2. The [CI system](https://travis-ci.org/Sage-Bionetworks/agora-data-manager) automatically updates the production DB
+2. The Github action CI system automatically updates the dev DB
 
 
 # Setup
 
-The following environment variables need to be setup for the scripts to deploy database updates:
+## Secrets
 
-| Variable             | Description                       | Example                                                                   |
-|----------------------|-----------------------------------|---------------------------------------------------------------------------|
-| BASTIAN_HOST_develop | The bastian host                  | ec2-10-11-12-13.compute-1.amazonaws.com                                   |
-| DB_HOST_develop      | The database host                 | dbcluster-mr0a782pfjnk.cluster-ctcayu3de2lt.us-east-1.docdb.amazonaws.com |
-| DB_USER_develop      | The database user                 | dbuser                                                                    |
-| DB_PASS_develop      | The database password             | supersecret                                                               |
-| SYNAPSE_USERNAME     | The Synapse service user          | syn-service-user                                                          |
-| SYNAPSE_PASSWORD     | The Synapse service user password | supersecret                                                               |
+The following secrets need to be setup in Github for the scripts to deploy database updates:
 
-__Note__: The variables containing `_develop` postfix corresponds to the branch.
-To deploy to a prod environment a prod branch is require along with a variable
-containing a `_prod` prefix (i.e. BASTIAN_HOST_prod)
+Global secrets:
 
+| Variable             | Description                       | Example                     |
+|----------------------|-----------------------------------|-----------------------------|
+| SYNAPSE_USERNAME     | The Synapse service user          | syn-service-user            |
+| SYNAPSE_PASSWORD     | The Synapse service user password | supersecret                 |
 
-[db_update]: diagram1.png "update diagram"
+
+Context specific secrets for each environment that corresponds to a git branch (develop/staging/prod):
+
+| Variable  | Description                 | Example                                                                   |
+|-----------|-----------------------------|---------------------------------------------------------------------------|
+| DB_HOST   | The database host           | dbcluster-mr0a782pfjnk.cluster-ctcayu3de2lt.us-east-1.docdb.amazonaws.com |
+| DB_USER   | The database user           | dbuser                                                                    |
+| DB_PASS   | The database password       | supersecret                                                               |
+
+
+![alt text][github_secrets]
+
+
+## Self hosted runners
+
+[agora2-infra] repository deploys a bastian host in AWS for each environment which have access to
+the databases.  We manually configure a [Github self-hosted runner] for each bastian host,
+a label is applied to each runner to match the corresponding deployment branch name (develop/staging/prod).
+Each runner corresponds to an environment which corresponds to a git branch. The update is
+executed from these runners.  When a push happens on a branch (i.e. develop), the update
+is executed on the `agora-bastian-develop` runner which in turn updates the development database.
+
+
+![alt text][self_hosted_runners]
+
+
+[db_update]: agora-db-update.drawio.png "update diagram"
+[github_secrets]: github_secrets.png "github secrets screen"
+[self_hosted_runners]: self-hosted-runners.png "self hosted runners"
+[agora2-infra]: https://github.com/Sage-Bionetworks/agora2-infra "agora2-infra repository"
+[Github self-hosted runners]: https://docs.github.com/en/actions/hosting-your-own-runners/managing-self-hosted-runners/about-self-hosted-runners#about-self-hosted-runners

data-manifest.json

@@ -1,5 +1,5 @@
 {
-    "data-version": "66",
+    "data-version": "68",
     "data-manifest-id": "syn13363290",
     "team-images-id": "syn12861877"
 }

import-data.sh

@@ -5,25 +5,25 @@
 #!/bin/bash
 set -e
 
-TRAVIS_BRANCH=$1
+BRANCH=$1
 SYNAPSE_USERNAME=$2
 SYNAPSE_PASSWORD=$3
 DB_HOST=$4
 DB_USER=$5
 DB_PASS=$6
 
 CURRENT_DIR=$(pwd)
-PARENT_DIR="$(dirname "$CURRENT_DIR")"
-TMP_DIR=/tmp
-WORKING_DIR=$TMP_DIR/work
+WORKING_DIR=$CURRENT_DIR
 DATA_DIR=$WORKING_DIR/data
 TEAM_IMAGES_DIR=$DATA_DIR/team_images
 
+mkdir -p $TEAM_IMAGES_DIR
+
 # Version key/value should be on his own line
 DATA_VERSION=$(cat $WORKING_DIR/data-manifest.json | grep data-version | head -1 | awk -F: '{ print $2 }' | sed 's/[",]//g' | tr -d '[[:space:]]')
 DATA_MANIFEST_ID=$(cat $WORKING_DIR/data-manifest.json | grep data-manifest-id | head -1 | awk -F: '{ print $2 }' | sed 's/[",]//g' | tr -d '[[:space:]]')
 TEAM_IMAGES_ID=$(cat $WORKING_DIR/data-manifest.json | grep team-images-id | head -1 | awk -F: '{ print $2 }' | sed 's/[",]//g' | tr -d '[[:space:]]')
-echo "$TRAVIS_BRANCH branch, DATA_VERSION = $DATA_VERSION, manifest id = $DATA_MANIFEST_ID"
+echo "$BRANCH branch, DATA_VERSION = $DATA_VERSION, manifest id = $DATA_MANIFEST_ID"
 
 # Download the manifest file from synapse
 synapse -u $SYNAPSE_USERNAME -p $SYNAPSE_PASSWORD get --downloadLocation $DATA_DIR -v $DATA_VERSION $DATA_MANIFEST_ID