From 4f3e40e0825a86cd2de065cc41661ab232486097 Mon Sep 17 00:00:00 2001 From: "George G. Vega Yon" Date: Tue, 21 Jan 2025 20:11:49 -0700 Subject: [PATCH 1/7] Inserting cfa-action --- .github/workflows/containers-and-az-pool.yaml | 89 +++---------------- 1 file changed, 10 insertions(+), 79 deletions(-) diff --git a/.github/workflows/containers-and-az-pool.yaml b/.github/workflows/containers-and-az-pool.yaml index 79ddf4cd..15b7078a 100644 --- a/.github/workflows/containers-and-az-pool.yaml +++ b/.github/workflows/containers-and-az-pool.yaml @@ -37,9 +37,9 @@ env: jobs: - build-dependencies-image: + build-pipeline-image: runs-on: cfa-cdcgov # VM based runner serving CFA's cdcgov repos (as opposed to cdcent) - name: Build dependencies image + name: Build simage outputs: tag: ${{ steps.image-tag.outputs.tag }} @@ -49,86 +49,17 @@ jobs: - name: Checkout code uses: actions/checkout@v4 - ######################################################################### - # Getting the tag from the branch - # The tag will be used for both the docker image and the batch pool - ######################################################################### - - # From: https://stackoverflow.com/a/58035262/2097171 - - name: Extract branch name - shell: bash - run: echo "branch=${GITHUB_HEAD_REF:-${GITHUB_REF#refs/heads/}}" >> $GITHUB_OUTPUT - id: branch-name - - - name: Figure out tag (either latest if it is main or the branch name) + - name: Build the images id: image-tag - run: | - if [ "${{ steps.branch-name.outputs.branch }}" = "main" ]; then - echo "tag=latest" >> $GITHUB_OUTPUT - else - echo "tag=${{ steps.branch-name.outputs.branch }}" >> $GITHUB_OUTPUT - fi - - # NOTE: This lookup is only for the cache _key_. We don't need the cache _value_ - # because we explicitly fetch image from the registry in the next step. Keeping the - # cached image on the runner causes the runner to quickly run out of storage. - - name: Check cache for base image - uses: actions/cache@v4 - id: cache + uses: CDCgov/cfa-actions/twostep-container-build@v1.2.0 with: - key: docker-dependencies-${{ runner.os }}-${{ hashFiles('./DESCRIPTION', './Dockerfile-dependencies') }}-${{ steps.image-tag.outputs.tag }} - lookup-only: true - path: - ./DESCRIPTION - - - name: Login to the Container Registry - if: steps.cache.outputs.cache-hit != 'true' - uses: docker/login-action@v3 - with: - registry: "cfaprdbatchcr.azurecr.io" - username: "cfaprdbatchcr" + container-file-1: ./Dockerfile-dependencies + container-file-2: ./Dockerfile + registry: cfaprdbatchcr.azurecr.io password: ${{ secrets.CFAPRDBATCHCR_REGISTRY_PASSWORD }} - - - name: Build and push - if: steps.cache.outputs.cache-hit != 'true' - uses: docker/build-push-action@v6 - with: - push: true - no-cache: true - tags: | - ${{ env.REGISTRY}}/${{ env.IMAGE_NAME }}-dependencies:${{ steps.image-tag.outputs.tag }} - file: ./Dockerfile-dependencies - - build-pipeline-image: - - name: Build pipeline image - - needs: build-dependencies-image - runs-on: cfa-cdcgov - - outputs: - tag: ${{ needs.build-dependencies-image.outputs.tag }} - - steps: - - - name: Login to the Container Registry - uses: docker/login-action@v3 - with: - registry: "cfaprdbatchcr.azurecr.io" - username: "cfaprdbatchcr" - password: ${{ secrets.CFAPRDBATCHCR_REGISTRY_PASSWORD }} - - - name: Build and push model pipeline image for Azure batch - id: build_and_push_model_image - uses: docker/build-push-action@v6 - with: - push: true # This can be toggled manually for tweaking. - no-cache: true - tags: | - ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:${{ needs.build-dependencies-image.outputs.tag }} - file: ./Dockerfile - build-args: | - TAG=${{ needs.build-dependencies-image.outputs.tag }} + username: cfaprdbatchcr + image: cfa-epinow2-pipeline + first-step-cache-key: docker-dependencies-${{ runner.os }}-${{ hashFiles('./DESCRIPTION', './Dockerfile-dependencies') }} batch-pool: From e4fbb7eb7a7c2d5b87ba0b28e1a31c2dcf14bb51 Mon Sep 17 00:00:00 2001 From: "George G. Vega Yon" Date: Tue, 21 Jan 2025 21:12:58 -0700 Subject: [PATCH 2/7] Update containers-and-az-pool.yaml --- .github/workflows/containers-and-az-pool.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/containers-and-az-pool.yaml b/.github/workflows/containers-and-az-pool.yaml index 15b7078a..55ba9012 100644 --- a/.github/workflows/containers-and-az-pool.yaml +++ b/.github/workflows/containers-and-az-pool.yaml @@ -32,7 +32,7 @@ on: env: # Together, these form: cfaprdbatchcr.azurecr.io/cfa-epinow2-pipeline - REGISTRY: cfaprdbatchcr.azurecr.io + REGISTRY: cfaprdbatchcr.azurecr.io/ IMAGE_NAME: cfa-epinow2-pipeline jobs: From 3fc433c950223d6f48d7fdc1d9b0ed53fb67b235 Mon Sep 17 00:00:00 2001 From: "George G. Vega Yon" Date: Tue, 21 Jan 2025 21:25:18 -0700 Subject: [PATCH 3/7] Fixing dependency --- .github/workflows/containers-and-az-pool.yaml | 2 +- Dockerfile | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/containers-and-az-pool.yaml b/.github/workflows/containers-and-az-pool.yaml index 55ba9012..f8eef5bf 100644 --- a/.github/workflows/containers-and-az-pool.yaml +++ b/.github/workflows/containers-and-az-pool.yaml @@ -39,7 +39,7 @@ jobs: build-pipeline-image: runs-on: cfa-cdcgov # VM based runner serving CFA's cdcgov repos (as opposed to cdcent) - name: Build simage + name: Build image outputs: tag: ${{ steps.image-tag.outputs.tag }} diff --git a/Dockerfile b/Dockerfile index d91baddf..37d0bb69 100644 --- a/Dockerfile +++ b/Dockerfile @@ -2,7 +2,7 @@ ARG TAG=local # This requires access to the Azure Container Registry -FROM cfaprdbatchcr.azurecr.io/cfa-epinow2-pipeline-dependencies:${TAG} +FROM cfaprdbatchcr.azurecr.io/cfa-epinow2-pipeline:${TAG} # Will copy the package to the container preserving the directory structure COPY . pkg/ From 9beb388e340f00894fe805310d5ce67d071710b2 Mon Sep 17 00:00:00 2001 From: "George G. Vega Yon" Date: Tue, 21 Jan 2025 21:28:25 -0700 Subject: [PATCH 4/7] Pointing to the wrong registry --- .github/workflows/containers-and-az-pool.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/containers-and-az-pool.yaml b/.github/workflows/containers-and-az-pool.yaml index f8eef5bf..780c192b 100644 --- a/.github/workflows/containers-and-az-pool.yaml +++ b/.github/workflows/containers-and-az-pool.yaml @@ -55,10 +55,10 @@ jobs: with: container-file-1: ./Dockerfile-dependencies container-file-2: ./Dockerfile - registry: cfaprdbatchcr.azurecr.io + registry: ${{ env.REGISTRY}} password: ${{ secrets.CFAPRDBATCHCR_REGISTRY_PASSWORD }} username: cfaprdbatchcr - image: cfa-epinow2-pipeline + image: ${{ env.IMAGE_NAME}} first-step-cache-key: docker-dependencies-${{ runner.os }}-${{ hashFiles('./DESCRIPTION', './Dockerfile-dependencies') }} batch-pool: From 58ade031966072f4a3a626551a93d180ae1b3e1b Mon Sep 17 00:00:00 2001 From: "George G. Vega Yon" Date: Wed, 22 Jan 2025 13:25:29 -0700 Subject: [PATCH 5/7] Updating the NEWS.md file --- NEWS.md | 1 + 1 file changed, 1 insertion(+) diff --git a/NEWS.md b/NEWS.md index 5149e6c7..a6bc6fd3 100644 --- a/NEWS.md +++ b/NEWS.md @@ -1,5 +1,6 @@ # CFAEpiNow2Pipeline (development version) +* Refactored container build workflow to use CDCgov/cfa-actions/twostep-container-build. * Fix Batch run trigger from Makefile and drop `test-` prefix * Add fit observation data to summaries * Use a consistent name scheme for geographies: `geo_value` From 6693ce16d2ec441d08adbd7ae3527005e7dd1280 Mon Sep 17 00:00:00 2001 From: "George G. Vega Yon" Date: Wed, 29 Jan 2025 11:27:15 -0700 Subject: [PATCH 6/7] Apply suggestions from code review Co-authored-by: Nate McIntosh --- .github/workflows/containers-and-az-pool.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/containers-and-az-pool.yaml b/.github/workflows/containers-and-az-pool.yaml index 780c192b..6f15aa15 100644 --- a/.github/workflows/containers-and-az-pool.yaml +++ b/.github/workflows/containers-and-az-pool.yaml @@ -55,10 +55,10 @@ jobs: with: container-file-1: ./Dockerfile-dependencies container-file-2: ./Dockerfile - registry: ${{ env.REGISTRY}} + registry: ${{ env.REGISTRY }} password: ${{ secrets.CFAPRDBATCHCR_REGISTRY_PASSWORD }} username: cfaprdbatchcr - image: ${{ env.IMAGE_NAME}} + image: ${{ env.IMAGE_NAME }} first-step-cache-key: docker-dependencies-${{ runner.os }}-${{ hashFiles('./DESCRIPTION', './Dockerfile-dependencies') }} batch-pool: From bd7412b2b36efa3b7205b9a0c5a043470ec1c7de Mon Sep 17 00:00:00 2001 From: Micah Wiesner Date: Fri, 14 Feb 2025 17:32:27 +0000 Subject: [PATCH 7/7] updating news --- NEWS.md | 1 + 1 file changed, 1 insertion(+) diff --git a/NEWS.md b/NEWS.md index 8b44d33a..0e00ac9e 100644 --- a/NEWS.md +++ b/NEWS.md @@ -1,6 +1,7 @@ # CFAEpiNow2Pipeline v0.2.0 ## Features +* Refactored GH Actions container build to cfa-actions 2-step build * Creating SOP.md to document weekly run procedures * Allows unique job_ids for runs. * Makefile supports either docker or podman as arguments to setup & manage containers