From 05d43e42d22e2c5532a9979a012af1c8ef564b98 Mon Sep 17 00:00:00 2001 From: Drew Yang Date: Wed, 17 Aug 2022 11:44:54 -0500 Subject: [PATCH] =?UTF-8?q?docs(SciOps=20Workflows):=20=F0=9F=93=9D=20add?= =?UTF-8?q?=20doc?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- doc/README.md | 38 +++++++++++++++++++ .../dist/alpine/codebook.Dockerfile | 31 +++++++++++++++ .../alpine/docker-compose-codebook_env.yaml | 17 +++++++++ .../dist/debian/codebook.Dockerfile | 29 ++++++++++++++ .../debian/docker-compose-codebook_env.yaml | 17 +++++++++ 5 files changed, 132 insertions(+) create mode 100644 doc/README.md create mode 100644 docker-template/u24_codebook_env/dist/alpine/codebook.Dockerfile create mode 100644 docker-template/u24_codebook_env/dist/alpine/docker-compose-codebook_env.yaml create mode 100644 docker-template/u24_codebook_env/dist/debian/codebook.Dockerfile create mode 100644 docker-template/u24_codebook_env/dist/debian/docker-compose-codebook_env.yaml diff --git a/doc/README.md b/doc/README.md new file mode 100644 index 0000000..073696e --- /dev/null +++ b/doc/README.md @@ -0,0 +1,38 @@ +# SciOps CICD Pipeline + +## Flow + +#### Rough Explanation +- Trigger: SciOps Workflow repo make push/PR/tag +- Codebook image + - Build Codebook image + - Publish Codebook image(only with repo tag push) +- Worker images + - Build CPU based worker images + - Publish worker images(only with repo tag push) + - GPU based worker image requires manually building and publishing on a dev GPU EC2 instance + + +#### Detailed Flow Diagram +![Image missing](./sciops_cicd.drawio.png) + +## Setup and Usage +- Enable your personal fork pipeline +- Add Github Actions Secrets to your personal fork settings + - BOT_SSH_KEY: sciops devops deploy key + - REGISTRY_USERNAME: private registry username + - REGISTRY_PASSWORD: private registry password +- Enable the main fork pipeline +- Add the same secrets to the main fork settings +- Do your development +- Push commits to your personal fork + - This will trigger your personal fork's pipeline to run, and it will only run through the image building part of the pipeline as a test before you make a pull request +- Make a pull request to the main fork + - This will trigger the main fork's pipeline to run, only run through the image building part as well to ensure the test before the pull request been merged +- Merge a pull request + - This will trigger the main fork's pipeline, but actually redundant +- Push a tag to the main fork + - This will trigger the main fork's pipeline and publish all the images to the private registry +- Update worker terraform image tag input + - Check instruction [here](https://github.com/yambottle/dj-gitops/tree/main/infrastructures/tf/sciops_workers) + diff --git a/docker-template/u24_codebook_env/dist/alpine/codebook.Dockerfile b/docker-template/u24_codebook_env/dist/alpine/codebook.Dockerfile new file mode 100644 index 0000000..270e2b0 --- /dev/null +++ b/docker-template/u24_codebook_env/dist/alpine/codebook.Dockerfile @@ -0,0 +1,31 @@ +## docker build --build-arg JHUB_VER=1.4.2 --build-arg PY_VER=3.8 --build-arg DIST=debian --build-arg DEPLOY_KEY=wt-ephys-no-curation-deploy.pem --build-arg REPO_OWNER=dj-sciops --build-arg REPO_NAME=wt-ephys-no-curation -f codebook.Dockerfile -t registry.vathes.com/sciops/codebook-wt-ephys-no-curation:v0.0.0 . + +## Single Stage +ARG JHUB_VER +ARG PY_VER +ARG DIST +ARG CODEBOOK_BASE_HASH +FROM datajoint/djlabhub:${JHUB_VER}-py${PY_VER}-${DIST}-${CODEBOOK_BASE_HASH} + +USER root +RUN apk add openssh git vim +USER anaconda + +ARG DEPLOY_KEY +COPY --chown=anaconda $DEPLOY_KEY $HOME/.ssh/id_ed25519 +RUN chmod u=r,g-rwx,o-rwx $HOME/.ssh/id_ed25519 + +ARG REPO_OWNER +ARG REPO_NAME +WORKDIR /tmp +RUN ssh-keyscan -t ed25519 github.com >> $HOME/.ssh/known_hosts && \ + git clone git@github.com:${REPO_OWNER}/${REPO_NAME}.git && \ + pip install ./${REPO_NAME} && \ + cp -r ./${REPO_NAME}/notebooks/ /home/ && \ + cp -r ./${REPO_NAME}/images/ /home/notebooks/ && \ + cp -r ./${REPO_NAME}/user_data/ /home/notebooks/ && \ + cp ./${REPO_NAME}/README.md /home/notebooks/ && \ + rm -rf /tmp/${REPO_NAME} && \ + rm -rf $HOME/.ssh/ + +WORKDIR /home/notebooks diff --git a/docker-template/u24_codebook_env/dist/alpine/docker-compose-codebook_env.yaml b/docker-template/u24_codebook_env/dist/alpine/docker-compose-codebook_env.yaml new file mode 100644 index 0000000..1db3661 --- /dev/null +++ b/docker-template/u24_codebook_env/dist/alpine/docker-compose-codebook_env.yaml @@ -0,0 +1,17 @@ +# cd ./docker-template/codebook_env && set -a && source .env && docker-compose -f dist/alpine/docker-compose-codebook_env.yaml build +version: '2.4' +services: + codebook_env: + build: + # only necessary if rebuilding image + context: ../../ + dockerfile: ./dist/alpine/codebook.Dockerfile + args: + - JHUB_VER + - PY_VER + - DIST + - CODEBOOK_BASE_HASH + - DEPLOY_KEY + - REPO_OWNER + - REPO_NAME + image: registry.vathes.com/sciops/codebook_${REPO_NAME}:jhub${JHUB_VER}-py${PY_VER}-${DIST}-${WORKFLOW_VERSION} diff --git a/docker-template/u24_codebook_env/dist/debian/codebook.Dockerfile b/docker-template/u24_codebook_env/dist/debian/codebook.Dockerfile new file mode 100644 index 0000000..32ecd43 --- /dev/null +++ b/docker-template/u24_codebook_env/dist/debian/codebook.Dockerfile @@ -0,0 +1,29 @@ +## docker build --build-arg JHUB_VER=1.4.2 --build-arg PY_VER=3.8 --build-arg DIST=debian --build-arg CODEBOOK_BASE_HASH=a7ee3ce --build-arg DEPLOY_KEY=wt-ephys-no-curation-deploy.pem --build-arg REPO_OWNER=dj-sciops --build-arg REPO_NAME=wt-ephys-no-curation -f codebook.Dockerfile -t registry.vathes.com/sciops/codebook-wt-ephys-no-curation:v0.0.0 . + +## Single Stage +ARG JHUB_VER +ARG PY_VER +ARG DIST +ARG CODEBOOK_BASE_HASH +FROM datajoint/djlabhub:${JHUB_VER}-py${PY_VER}-${DIST}-${CODEBOOK_BASE_HASH} + +ARG DEPLOY_KEY +COPY --chown=anaconda $DEPLOY_KEY $HOME/.ssh/id_ed25519 +RUN chmod u=r,g-rwx,o-rwx $HOME/.ssh/id_ed25519 && \ + printf "ssh\ngit" >> /tmp/apt_requirements.txt && \ + /entrypoint.sh echo "installed" + +ARG REPO_OWNER +ARG REPO_NAME +WORKDIR /tmp +RUN ssh-keyscan -t ed25519 github.com >> $HOME/.ssh/known_hosts && \ + git clone git@github.com:${REPO_OWNER}/${REPO_NAME}.git && \ + pip install ./${REPO_NAME} && \ + cp -r ./${REPO_NAME}/notebooks/ /home/ && \ + cp -r ./${REPO_NAME}/images/ /home/notebooks/ || true && \ + cp -r ./${REPO_NAME}/user_data/ /home/ && \ + cp ./${REPO_NAME}/README.md /home/notebooks/ && \ + rm -rf /tmp/${REPO_NAME} && \ + rm -rf $HOME/.ssh/ + +WORKDIR /home/notebooks diff --git a/docker-template/u24_codebook_env/dist/debian/docker-compose-codebook_env.yaml b/docker-template/u24_codebook_env/dist/debian/docker-compose-codebook_env.yaml new file mode 100644 index 0000000..cdda174 --- /dev/null +++ b/docker-template/u24_codebook_env/dist/debian/docker-compose-codebook_env.yaml @@ -0,0 +1,17 @@ +# cd ./docker-template/codebook_env && set -a && source .env && docker-compose -f dist/debian/docker-compose-codebook_env.yaml build +version: '2.4' +services: + codebook_env: + build: + # only necessary if rebuilding image + context: ../../ + dockerfile: ./dist/debian/codebook.Dockerfile + args: + - JHUB_VER + - PY_VER + - DIST + - CODEBOOK_BASE_HASH + - DEPLOY_KEY + - REPO_OWNER + - REPO_NAME + image: registry.vathes.com/sciops/codebook_${REPO_NAME}:jhub${JHUB_VER}-py${PY_VER}-${DIST}-${WORKFLOW_VERSION}