Merge pull request #26 from aodn/task_definition_simplify

deploy: container vars refactoring
aodn · Dec 13, 2023 · e0831bf · e0831bf
2 parents 158915c + 9c8875a
commit e0831bf
Show file tree

Hide file tree

Showing 17 changed files with 376 additions and 324 deletions.
diff --git a/.github/workflows/deploy-development.yml b/.github/workflows/deploy-development.yml
@@ -0,0 +1,131 @@
+name: Build, Test and Deploy Development
+
+on:
+  pull_request:
+    branches:
+      - master
+
+permissions:
+  id-token: write   # This is required for requesting the JWT
+  contents: read    # This is required for actions/checkout
+
+jobs:
+  build_test_push:
+    runs-on: ubuntu-latest
+    environment: development
+    outputs:
+      image_tag: ${{ steps.set_image_tag.outputs.image_tag }}
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+
+      - name: Set up QEMU
+        uses: docker/setup-qemu-action@v3
+
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v3
+
+      - name: Setup Docker Structure Test
+        run: >
+          curl -LO
+          https://storage.googleapis.com/container-structure-test/latest/container-structure-test-linux-amd64
+          && chmod +x container-structure-test-linux-amd64 && sudo mv container-structure-test-linux-amd64
+          /usr/local/bin/container-structure-test
+
+      - name: Set Image Tag
+        id: set_image_tag
+        run: |
+          branch_name=${{ github.head_ref || github.ref_name }}
+          tag=${{ env.TAG_PREFIX}}-${branch_name//\//-}
+          echo "$tag"
+          echo "image_tag=$tag" >> $GITHUB_OUTPUT
+        env:
+          TAG_PREFIX: dev
+
+      - name: Build Docker Image
+        uses: docker/build-push-action@v5
+        with:
+          context: .
+          load: true
+          tags: ${{ vars.ECR_REPOSITORY }}:${{ steps.set_image_tag.outputs.image_tag }}
+
+      - name: Test Docker Image
+        run: |
+          container-structure-test test --image ${{ vars.ECR_REPOSITORY }}:${{ steps.set_image_tag.outputs.image_tag }} --config tests/config.yaml
+
+      - name: Configure AWS Credentials
+        uses: aws-actions/configure-aws-credentials@v4
+        with:
+          audience: sts.amazonaws.com
+          aws-region: ${{ vars.AWS_REGION }}
+          role-to-assume: ${{ secrets.AWS_ROLE_ARN }}
+
+      - name: Login to ECR
+        uses: docker/login-action@v3
+        with:
+          registry: ${{ vars.ECR_REGISTRY }}
+
+      - name: Build and Push Docker Image
+        id: build_and_push
+        uses: docker/build-push-action@v5
+        with:
+          context: .
+#          Only building for AMD64 for now
+#          platforms: linux/amd64,linux/arm64
+          push: true
+          tags: ${{ vars.ECR_REGISTRY }}/${{ vars.ECR_REPOSITORY }}:${{ steps.set_image_tag.outputs.image_tag }}
+
+  development_deploy:
+    runs-on: ubuntu-latest
+    environment: development
+    env:
+      tf_version: '1.5.7'
+      tg_version: '0.54.0'
+      tg_dir: './deploy/tg'
+    needs: [build_test_push]
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+
+      - name: Configure AWS Credentials
+        uses: aws-actions/configure-aws-credentials@v4
+        with:
+          audience: sts.amazonaws.com
+          aws-region: ${{ vars.AWS_REGION }}
+          role-to-assume: ${{ secrets.AWS_ROLE_ARN }}
+
+      - name: Expose github environment as shell variables
+        env:
+          SECRETS_CONTEXT: ${{ toJson(secrets) }}
+          VARS_CONTEXT: ${{ toJson(vars) }}
+        run: |
+          EOF=$(dd if=/dev/urandom bs=15 count=1 status=none | base64)
+          to_envs() { jq -r "to_entries[] | \"\(.key)<<$EOF\n\(.value)\n$EOF\n\""; }
+          echo "$VARS_CONTEXT" | to_envs >> $GITHUB_ENV
+          echo "$SECRETS_CONTEXT" | to_envs >> $GITHUB_ENV
+
+      - name: Terragrunt Plan
+        uses: gruntwork-io/terragrunt-action@v2
+        with:
+          tf_version: ${{ env.tf_version }}
+          tg_version: ${{ env.tg_version }}
+          tg_dir: ${{ env.tg_dir }}
+          tg_command: 'plan -out=tf.plan'
+        env:
+          TF_INPUT: 0
+          TF_IN_AUTOMATION: true
+          # get the image digest from the build job with optional override from vars context
+          TF_VAR_image: ${{ vars.IMAGE || needs.build_test_push.outputs.image_tag }}
+
+      - name: Terragrunt Apply
+        uses: gruntwork-io/terragrunt-action@v2
+        with:
+          tf_version: ${{ env.tf_version }}
+          tg_version: ${{ env.tg_version }}
+          tg_dir: ${{ env.tg_dir }}
+          tg_command: '--terragrunt-non-interactive --terragrunt-log-level info apply -auto-approve tf.plan'
+        env:
+          TF_INPUT: 0
+          TF_IN_AUTOMATION: true
+          # get the image digest from the build job with optional override from vars context
+          TF_VAR_image: ${{ vars.IMAGE || needs.build_test_push.outputs.image_tag }}
diff --git a/.github/workflows/deploy-production.yml b/.github/workflows/deploy-production.yml
@@ -83,35 +83,26 @@ jobs:
           echo "$VARS_CONTEXT" | to_envs >> $GITHUB_ENV
           echo "$SECRETS_CONTEXT" | to_envs >> $GITHUB_ENV
 
-      - name: Setup Terraform
-        uses: hashicorp/setup-terraform@v3
-        with:
-          terraform_version: ${{ env.tf_version }}
-
-      - name: Setup Terragrunt
-        id: setup_terragrunt
-        run: |
-          wget https://github.com/gruntwork-io/terragrunt/releases/download/v${terragrunt_version}/terragrunt_linux_amd64 \
-            && mv terragrunt_linux_amd64 terragrunt \
-            && chmod +x terragrunt \
-            && mv terragrunt /usr/local/bin/terragrunt
-        env:
-          terragrunt_version: ${{ env.tg_version }}
-
       - name: Terragrunt Plan
-        id: terragrunt_plan
-        run: terragrunt plan -out=tf.plan
-        working-directory: ${{ env.tg_dir }}
+        uses: gruntwork-io/terragrunt-action@v2
+        with:
+          tf_version: ${{ env.tf_version }}
+          tg_version: ${{ env.tg_version }}
+          tg_dir: ${{ env.tg_dir }}
+          tg_command: 'plan -out=tf.plan'
         env:
           TF_INPUT: 0
           TF_IN_AUTOMATION: true
           # get the image digest from the build job with optional override from vars context
           TF_VAR_image: ${{ vars.IMAGE || needs.build_push.outputs.image_digest }}
 
       - name: Terragrunt Apply
-        id: terragrunt_apply
-        run: terragrunt apply -auto-approve tf.plan
-        working-directory: ${{ env.tg_dir }}
+        uses: gruntwork-io/terragrunt-action@v2
+        with:
+          tf_version: ${{ env.tf_version }}
+          tg_version: ${{ env.tg_version }}
+          tg_dir: ${{ env.tg_dir }}
+          tg_command: '--terragrunt-non-interactive --terragrunt-log-level info apply -auto-approve tf.plan'
         env:
           TF_INPUT: 0
           TF_IN_AUTOMATION: true

diff --git a/.github/workflows/deploy-staging.yml b/.github/workflows/deploy-staging.yml
@@ -77,35 +77,26 @@ jobs:
           echo "$VARS_CONTEXT" | to_envs >> $GITHUB_ENV
           echo "$SECRETS_CONTEXT" | to_envs >> $GITHUB_ENV
 
-      - name: Setup Terraform
-        uses: hashicorp/setup-terraform@v3
-        with:
-          terraform_version: ${{ env.tf_version }}
-
-      - name: Setup Terragrunt
-        id: setup_terragrunt
-        run: |
-          wget https://github.com/gruntwork-io/terragrunt/releases/download/v${terragrunt_version}/terragrunt_linux_amd64 \
-            && mv terragrunt_linux_amd64 terragrunt \
-            && chmod +x terragrunt \
-            && mv terragrunt /usr/local/bin/terragrunt
-        env:
-          terragrunt_version: ${{ env.tg_version }}
-
       - name: Terragrunt Plan
-        id: terragrunt_plan
-        run: terragrunt plan -out=tf.plan
-        working-directory: ${{ env.tg_dir }}
+        uses: gruntwork-io/terragrunt-action@v2
+        with:
+          tf_version: ${{ env.tf_version }}
+          tg_version: ${{ env.tg_version }}
+          tg_dir: ${{ env.tg_dir }}
+          tg_command: 'plan -out=tf.plan'
         env:
           TF_INPUT: 0
           TF_IN_AUTOMATION: true
           # get the image digest from the build job with optional override from vars context
           TF_VAR_image: ${{ vars.IMAGE || needs.build_push.outputs.image_digest }}
 
       - name: Terragrunt Apply
-        id: terragrunt_apply
-        run: terragrunt apply -auto-approve tf.plan
-        working-directory: ${{ env.tg_dir }}
+        uses: gruntwork-io/terragrunt-action@v2
+        with:
+          tf_version: ${{ env.tf_version }}
+          tg_version: ${{ env.tg_version }}
+          tg_dir: ${{ env.tg_dir }}
+          tg_command: '--terragrunt-non-interactive --terragrunt-log-level info apply -auto-approve tf.plan'
         env:
           TF_INPUT: 0
           TF_IN_AUTOMATION: true

diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml
diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml
@@ -18,6 +18,7 @@ repos:
     rev: v1.83.4
     hooks:
       - id: terraform_fmt
+      - id: terragrunt_fmt
       - id: terraform_validate
         args:
           - --tf-init-args=-backend=false

diff --git a/deploy/container/vars.yaml b/deploy/container/vars.yaml
@@ -0,0 +1,10 @@
+allowed_hosts: "*"
+allowed_cidr_nets: "0.0.0.0/0"
+django_secret_key: changeme
+db_host: db_host
+db_name: app_db
+db_user: app_db
+db_secret_name: /my/db/secret
+db_secret_region: ap-southeast-2
+s3_storage_bucket_name: appbucket
+s3_storage_bucket_region: ap-southeast-2
diff --git a/deploy/github/dev.env b/deploy/github/dev.env
diff --git a/deploy/github/development.env b/deploy/github/development.env
@@ -0,0 +1,18 @@
+# general environment variables for Terragrunt
+ALB_PARAMETER_NAME=shared-alb-devops-sydney
+APP_NAME=sample-django-app
+AWS_ACCOUNT_ID=450356697252
+AWS_REGION=ap-southeast-2
+ECR_REGISTRY=450356697252.dkr.ecr.ap-southeast-2.amazonaws.com
+ECR_REPOSITORY=api
+ENVIRONMENT=development
+
+# container definition variables
+ALLOWED_CIDR_NETS=10.32.48.0/20,10.32.64.0/20,10.32.80.0/20
+DB_HOST=stefan-db-rds-primary-evaluation.gamma.aodn.org.au
+DB_NAME=api_dev
+DB_SECRET_NAME=/rds/stefan-db/primary/evaluation/api
+DB_SECRET_REGION=ap-southeast-2
+DB_USER=api_dev
+S3_STORAGE_BUCKET_NAME=sample-django-app-development-450356697252
+S3_STORAGE_BUCKET_REGION=ap-southeast-2
diff --git a/deploy/github/production.env b/deploy/github/production.env
@@ -1,11 +1,18 @@
+# general environment variables for Terragrunt
 ALB_PARAMETER_NAME=shared-alb-devops-sydney
 APP_NAME=sample-django-app
 AWS_ACCOUNT_ID=450356697252
 AWS_REGION=ap-southeast-2
-DB_NAME=api_prod
-DB_USER=api_prod
-ECR_PARAMETER_NAME=api
 ECR_REGISTRY=450356697252.dkr.ecr.ap-southeast-2.amazonaws.com
 ECR_REPOSITORY=api
 ENVIRONMENT=production
-RDS_PARAMETER_NAME=stefan-db/primary/evaluation
+
+# container definition variables
+ALLOWED_CIDR_NETS=10.32.48.0/20,10.32.64.0/20,10.32.80.0/20
+DB_HOST=stefan-db-rds-primary-evaluation.gamma.aodn.org.au
+DB_NAME=api_prod
+DB_SECRET_NAME=/rds/stefan-db/primary/evaluation/api
+DB_SECRET_REGION=ap-southeast-2
+DB_USER=api_prod
+S3_STORAGE_BUCKET_NAME=sample-django-app-production-450356697252
+S3_STORAGE_BUCKET_REGION=ap-southeast-2
diff --git a/deploy/github/staging.env b/deploy/github/staging.env
@@ -1,9 +1,18 @@
+# general environment variables for Terragrunt
 ALB_PARAMETER_NAME=shared-alb-devops-sydney
 APP_NAME=sample-django-app
 AWS_ACCOUNT_ID=450356697252
 AWS_REGION=ap-southeast-2
-ECR_PARAMETER_NAME=api
 ECR_REGISTRY=450356697252.dkr.ecr.ap-southeast-2.amazonaws.com
 ECR_REPOSITORY=api
 ENVIRONMENT=staging
-RDS_PARAMETER_NAME=stefan-db/primary/evaluation
+
+# container definition variables
+ALLOWED_CIDR_NETS=10.32.48.0/20,10.32.64.0/20,10.32.80.0/20
+DB_HOST=stefan-db-rds-primary-evaluation.gamma.aodn.org.au
+DB_NAME=api
+DB_SECRET_NAME=/rds/stefan-db/primary/evaluation/api
+DB_SECRET_REGION=ap-southeast-2
+DB_USER=api
+S3_STORAGE_BUCKET_NAME=sample-django-app-staging-450356697252
+S3_STORAGE_BUCKET_REGION=ap-southeast-2
diff --git a/deploy/tf/alb.tf b/deploy/tf/alb.tf
@@ -1,6 +1,6 @@
 resource "aws_lb_target_group" "app" {
   name        = "${var.app_name}-${var.environment}"
-  port        = 80
+  port        = var.nginx_proxy ? var.proxy_port : var.app_port
   protocol    = "HTTP"
   target_type = "ip"
   vpc_id      = local.vpc_id