From 02d730c764b54a0821b6be5c6645224cf8e8eb12 Mon Sep 17 00:00:00 2001 From: Stefan Hattrell <29941279+digorgonzola@users.noreply.github.com> Date: Tue, 5 Dec 2023 19:10:56 +1100 Subject: [PATCH 01/55] include tg fmt in pre-commit hooks --- .pre-commit-config.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index f468b2a..efd206f 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -19,3 +19,4 @@ repos: hooks: - id: terraform_fmt - id: terraform_validate + - id: terragrunt_fmt From 420af2c33c7e2a1bb9a1fbb3f8e59ca9febe2578 Mon Sep 17 00:00:00 2001 From: Stefan Hattrell <29941279+digorgonzola@users.noreply.github.com> Date: Tue, 5 Dec 2023 19:11:31 +1100 Subject: [PATCH 02/55] .gitignore: include .terraform.lock.hcl files --- .gitignore | 1 - 1 file changed, 1 deletion(-) diff --git a/.gitignore b/.gitignore index 2527c5e..266a32f 100644 --- a/.gitignore +++ b/.gitignore @@ -110,4 +110,3 @@ venv.bak/ **/.terraform/* *.tfstate *.tfstate.* -.terraform.lock.hcl From cc3f04b99d381281d86ba3c585b8e8424facf236 Mon Sep 17 00:00:00 2001 From: Stefan Hattrell <29941279+digorgonzola@users.noreply.github.com> Date: Tue, 5 Dec 2023 19:12:14 +1100 Subject: [PATCH 03/55] add terragrunt deployment of ecs components also include s3 buckets --- deploy/tf/ecs/alb.tf | 41 +++++++ deploy/tf/ecs/ecs.tf | 180 ++++++++++++++++++++++++++++++ deploy/tf/ecs/main.tf | 0 deploy/tf/ecs/outputs.tf | 3 + deploy/tf/ecs/parameters.tf | 7 ++ deploy/tf/ecs/variables.tf | 35 ++++++ deploy/tg/ecs/.terraform.lock.hcl | 24 ++++ deploy/tg/ecs/terragrunt.hcl | 33 ++++++ deploy/tg/s3/.terraform.lock.hcl | 24 ++++ deploy/tg/s3/terragrunt.hcl | 27 +++++ deploy/tg/terragrunt.hcl | 53 +++++++++ 11 files changed, 427 insertions(+) create mode 100644 deploy/tf/ecs/alb.tf create mode 100644 deploy/tf/ecs/ecs.tf create mode 100644 deploy/tf/ecs/main.tf create mode 100644 deploy/tf/ecs/outputs.tf create mode 100644 deploy/tf/ecs/parameters.tf create mode 100644 deploy/tf/ecs/variables.tf create mode 100644 deploy/tg/ecs/.terraform.lock.hcl create mode 100644 deploy/tg/ecs/terragrunt.hcl create mode 100644 deploy/tg/s3/.terraform.lock.hcl create mode 100644 deploy/tg/s3/terragrunt.hcl create mode 100644 deploy/tg/terragrunt.hcl diff --git a/deploy/tf/ecs/alb.tf b/deploy/tf/ecs/alb.tf new file mode 100644 index 0000000..b0f154a --- /dev/null +++ b/deploy/tf/ecs/alb.tf @@ -0,0 +1,41 @@ +resource "aws_lb_target_group" "app" { + name = "sample-django-app" + port = 80 + protocol = "HTTP" + target_type = "ip" + vpc_id = local.params.vpc_id + + health_check { + enabled = true + path = "/health" + } +} + +resource "aws_route53_record" "app" { + for_each = toset(var.app_hostnames) + zone_id = local.params.dns_zone_id + name = each.value + type = "A" + + alias { + name = local.params.alb_dns_name + zone_id = local.params.alb_zone_id + evaluate_target_health = true + } +} + +resource "aws_lb_listener_rule" "app_fgate" { + for_each = toset(var.app_hostnames) + listener_arn = local.params.alb_https_listener + + action { + type = "forward" + target_group_arn = aws_lb_target_group.app.arn + } + + condition { + host_header { + values = [aws_route53_record.app[each.value].fqdn] + } + } +} diff --git a/deploy/tf/ecs/ecs.tf b/deploy/tf/ecs/ecs.tf new file mode 100644 index 0000000..f61e660 --- /dev/null +++ b/deploy/tf/ecs/ecs.tf @@ -0,0 +1,180 @@ +module "ecs" { + source = "terraform-aws-modules/ecs/aws" + version = "~> 5.7.0" + + # Cluster Configuration + cluster_name = "sample-django-app" + cluster_configuration = { + name = "containerInsights" + value = "enabled" + } + create_task_exec_iam_role = true + fargate_capacity_providers = { + FARGATE = { + default_capacity_provider_strategy = { + weight = 50 + } + } + FARGATE_SPOT = { + default_capacity_provider_strategy = { + weight = 50 + } + } + } + + # Service Configuration + services = { + + sample-django-app = { + capacity_provider_strategy = { + dedicated = { + base = 0 + capacity_provider = "FARGATE" + weight = 100 + } + # spot = { + # base = 0 + # capacity_provider = "FARGATE_SPOT" + # weight = 100 + # } + } + + # allow ECS exec commands on containers (e.g. to get a shell session) + enable_execute_command = true + + # resources + cpu = 512 + memory = 1024 + + # Container definition(s) + container_definitions = { + api = { + name = "api" + image = "${var.ecr_registry}/api:${var.image_tag}" + health_check = { + command = ["CMD-SHELL", "uwsgi-is-ready --stats-socket /tmp/statsock > /dev/null 2>&1 || exit 1"] + } + readonly_root_filesystem = false + essential = true + memory_reservation = 256 + environment = [ + { name = "DJANGO_SECRET_KEY", value = "changeme" }, + { name = "DB_HOST", value = local.params.rds_url }, + { name = "DB_NAME", value = "api" }, + { name = "DB_USER", value = "api" }, + { name = "DB_SECRET_NAME", value = var.db_secret_name }, + { name = "DB_SECRET_REGION", value = var.db_secret_region }, + { name = "ALLOWED_HOSTS", value = "*" }, + { name = "ALLOWED_CIDR_NETS", value = "${join(",", local.params.vpc_subnet_cidrs.private)}" }, + { name = "S3_STORAGE_BUCKET_NAME", value = var.s3_buckets["sample-django-app-bucket"].s3_bucket_id }, + { name = "S3_STORAGE_BUCKET_REGION", value = var.s3_buckets["sample-django-app-bucket"].s3_bucket_region } + ] + port_mappings = [ + { + name = "api" + containerPort = 9000 + hostPort = 9000 + } + ] + mount_points = [ + { + readOnly = false + containerPath = "/vol/web" + sourceVolume = "static" + } + ] + } + proxy = { + name = "proxy" + image = "${var.ecr_registry}/nginx-proxy:${var.image_tag}" + health_check = { + command = ["CMD-SHELL", "curl -so /dev/null http://localhost/health || exit 1"] + } + readonly_root_filesystem = false + essential = true + memory_reservation = 256 + environment = [ + { name = "APP_HOST", value = "127.0.0.1" }, + { name = "APP_PORT", value = 9000 }, + { name = "LISTEN_PORT", value = 80 } + ] + port_mappings = [ + { + name = "nginx" + containerPort = 80 + hostPort = 80 + } + ] + mount_points = [ + { + readOnly = false + containerPath = "/vol/static" + sourceVolume = "static" + } + ] + } + } + + deployment_circuit_breaker = { + enable = true + rollback = true + } + + load_balancer = { + service = { + target_group_arn = aws_lb_target_group.app.arn + container_name = "proxy" + container_port = 80 + } + } + + subnet_ids = local.params.vpc_private_subnets + + security_group_rules = { + ingress_vpc = { + type = "ingress" + from_port = 80 + to_port = 80 + protocol = "tcp" + cidr_blocks = [local.params.vpc_cidr] + } + egress_all = { + type = "egress" + from_port = 0 + to_port = 0 + protocol = "-1" + cidr_blocks = ["0.0.0.0/0"] + } + } + + tasks_iam_role_statements = [ + { + actions = [ + "s3:PutObject", + "s3:GetObjectAcl", + "s3:GetObject", + "s3:ListBucket", + "s3:DeleteObject", + "s3:PutObjectAcl" + ] + resources = [ + var.s3_buckets["sample-django-app-bucket"].s3_bucket_arn, + "${var.s3_buckets["sample-django-app-bucket"].s3_bucket_arn}/*" + ] + }, + { + actions = [ + "secretsmanager:GetSecretValue" + ] + resources = [ + "arn:aws:secretsmanager:ap-southeast-2:450356697252:secret:/rds/stefan-db/primary/evaluation/api*" + ] + } + ] + + volume = { + static = {} + } + } + } +} diff --git a/deploy/tf/ecs/main.tf b/deploy/tf/ecs/main.tf new file mode 100644 index 0000000..e69de29 diff --git a/deploy/tf/ecs/outputs.tf b/deploy/tf/ecs/outputs.tf new file mode 100644 index 0000000..b11c25e --- /dev/null +++ b/deploy/tf/ecs/outputs.tf @@ -0,0 +1,3 @@ +#output "params" { +# value = nonsensitive(local.params) +#} diff --git a/deploy/tf/ecs/parameters.tf b/deploy/tf/ecs/parameters.tf new file mode 100644 index 0000000..290f69c --- /dev/null +++ b/deploy/tf/ecs/parameters.tf @@ -0,0 +1,7 @@ +locals { + params = jsondecode(data.aws_ssm_parameter.shared.value) +} + +data "aws_ssm_parameter" "shared" { + name = var.parameter_name +} diff --git a/deploy/tf/ecs/variables.tf b/deploy/tf/ecs/variables.tf new file mode 100644 index 0000000..78c7589 --- /dev/null +++ b/deploy/tf/ecs/variables.tf @@ -0,0 +1,35 @@ +variable "app_hostnames" { + description = "Hostnames to associate with the application" + type = list(string) +} + +variable "db_secret_name" { + description = "The name of the secret to fetch DB login credentials from" + type = string +} + +variable "db_secret_region" { + description = "The region to fetch the secret from" + type = string + default = "ap-southeast-2" +} + +variable "ecr_registry" { + description = "The URL of the docker registry" + type = string +} + +variable "image_tag" { + description = "The tag of the docker image to pull from ECR" + type = string + default = "latest" +} + +variable "s3_buckets" { + description = "Map of s3 buckets" + type = any +} + +variable "parameter_name" { + type = string +} diff --git a/deploy/tg/ecs/.terraform.lock.hcl b/deploy/tg/ecs/.terraform.lock.hcl new file mode 100644 index 0000000..cdc06e6 --- /dev/null +++ b/deploy/tg/ecs/.terraform.lock.hcl @@ -0,0 +1,24 @@ +# This file is maintained automatically by "terraform init". +# Manual edits may be lost in future updates. + +provider "registry.terraform.io/hashicorp/aws" { + version = "5.29.0" + hashes = [ + "h1:SyiKAX/D3ZE9My7P03DrRMf65pNnfSDQXPb0g11lCS0=", + "zh:0453c1c64e51cd7050ce46d9280a0195b9073592508077ebf1a1c45f7026f3f5", + "zh:3ee87d1a2870b61fdcc80f3f96b669dbcc8171aadb821bec0e1fa0e6fb9595b6", + "zh:423c0304eba345167cc37dcd300712f24f03fe4de8eecc15edb0d4f88b29ec79", + "zh:6816ce0ed702263297a8e02467bb712c509a9f6e4f132a152a10f1cc19191a81", + "zh:6feb8a0aedabd778216238e72273f5c2ee86d8841acc3fb3dc9d8014a2bbdc51", + "zh:709ccdc8b37f975d422e7955814671548887613931e234e06249da629b0f2f95", + "zh:76c55744020dbdafea25be634f8ac37c1e371f8c397f73bd89bc270d00ee0834", + "zh:7e48d6fc488b9dbe2fd4bebefa1b485d04da38b11a6799f8cba178173b7f8782", + "zh:951d7ef2adbfb96b1d3e9c4780b2ab4375caf9c6b522a2d023c02ff0698d8e2a", + "zh:9b12af85486a96aedd8d7984b0ff811a4b42e3d88dad1a3fb4c0b580d04fa425", + "zh:b0bf5974bc1a7d2ce3f3a9a31a8238ad15ad02211f1e84c54832541ec4bd5d10", + "zh:cc56d4ab9bcbee95f73dbe90f11d4ff7299b835dddf2b30cfda526a2cccd0f9f", + "zh:cfe3a4394f2f7044e03bb63f4fb9c691926607c6784417ac9c0724943da60d09", + "zh:d6f82e13f33f70de8df480287b5a961ced5606f041d1c589f706b112f68db890", + "zh:fb7be5bcff62d0ca9edd4a1bee4d2ed16e9428e3f9eff3ea4d898ecb234505a3", + ] +} diff --git a/deploy/tg/ecs/terragrunt.hcl b/deploy/tg/ecs/terragrunt.hcl new file mode 100644 index 0000000..f102ae8 --- /dev/null +++ b/deploy/tg/ecs/terragrunt.hcl @@ -0,0 +1,33 @@ +dependency "s3" { + config_path = "../s3" + mock_outputs = { + wrapper = { + sample-django-app-bucket = { + s3_bucket_arn = "arn:aws:s3:::sample-django-app-123456789012" + s3_bucket_id = "sample-django-app-123456789012" + s3_bucket_region = "ap-southeast-2" + } + } + } +} + +include "global" { + path = find_in_parent_folders("terragrunt.hcl") + expose = true +} + +inputs = { + app_hostnames = ["api", "sample-django-app"] + db_secret_name = "/rds/stefan-db/primary/evaluation/api" + parameter_name = get_env("PARAMETER_NAME", "/apps/shared/devops/sydney") + ecr_registry = get_env("ECR_REGISTRY", "450356697252.dkr.ecr.ap-southeast-2.amazonaws.com") + s3_buckets = dependency.s3.outputs.wrapper +} + +locals { + global = include.global.locals +} + +terraform { + source = "${get_repo_root()}//deploy/tf/ecs" +} diff --git a/deploy/tg/s3/.terraform.lock.hcl b/deploy/tg/s3/.terraform.lock.hcl new file mode 100644 index 0000000..cdc06e6 --- /dev/null +++ b/deploy/tg/s3/.terraform.lock.hcl @@ -0,0 +1,24 @@ +# This file is maintained automatically by "terraform init". +# Manual edits may be lost in future updates. + +provider "registry.terraform.io/hashicorp/aws" { + version = "5.29.0" + hashes = [ + "h1:SyiKAX/D3ZE9My7P03DrRMf65pNnfSDQXPb0g11lCS0=", + "zh:0453c1c64e51cd7050ce46d9280a0195b9073592508077ebf1a1c45f7026f3f5", + "zh:3ee87d1a2870b61fdcc80f3f96b669dbcc8171aadb821bec0e1fa0e6fb9595b6", + "zh:423c0304eba345167cc37dcd300712f24f03fe4de8eecc15edb0d4f88b29ec79", + "zh:6816ce0ed702263297a8e02467bb712c509a9f6e4f132a152a10f1cc19191a81", + "zh:6feb8a0aedabd778216238e72273f5c2ee86d8841acc3fb3dc9d8014a2bbdc51", + "zh:709ccdc8b37f975d422e7955814671548887613931e234e06249da629b0f2f95", + "zh:76c55744020dbdafea25be634f8ac37c1e371f8c397f73bd89bc270d00ee0834", + "zh:7e48d6fc488b9dbe2fd4bebefa1b485d04da38b11a6799f8cba178173b7f8782", + "zh:951d7ef2adbfb96b1d3e9c4780b2ab4375caf9c6b522a2d023c02ff0698d8e2a", + "zh:9b12af85486a96aedd8d7984b0ff811a4b42e3d88dad1a3fb4c0b580d04fa425", + "zh:b0bf5974bc1a7d2ce3f3a9a31a8238ad15ad02211f1e84c54832541ec4bd5d10", + "zh:cc56d4ab9bcbee95f73dbe90f11d4ff7299b835dddf2b30cfda526a2cccd0f9f", + "zh:cfe3a4394f2f7044e03bb63f4fb9c691926607c6784417ac9c0724943da60d09", + "zh:d6f82e13f33f70de8df480287b5a961ced5606f041d1c589f706b112f68db890", + "zh:fb7be5bcff62d0ca9edd4a1bee4d2ed16e9428e3f9eff3ea4d898ecb234505a3", + ] +} diff --git a/deploy/tg/s3/terragrunt.hcl b/deploy/tg/s3/terragrunt.hcl new file mode 100644 index 0000000..51a6d79 --- /dev/null +++ b/deploy/tg/s3/terragrunt.hcl @@ -0,0 +1,27 @@ +include "global" { + path = find_in_parent_folders("terragrunt.hcl") + expose = true +} + +inputs = { + items = { + sample-django-app-bucket = { + bucket = "sample-django-app-${local.global.aws_account}" + acl = "public-read" + block_public_acls = false + block_public_policy = false + ignore_public_acls = false + control_object_ownership = true + object_ownership = "BucketOwnerPreferred" + restrict_public_buckets = false + } + } +} + +locals { + global = include.global.locals +} + +terraform { + source = "tfr:///terraform-aws-modules/s3-bucket/aws//wrappers?version=3.15.1" +} diff --git a/deploy/tg/terragrunt.hcl b/deploy/tg/terragrunt.hcl new file mode 100644 index 0000000..ef2f408 --- /dev/null +++ b/deploy/tg/terragrunt.hcl @@ -0,0 +1,53 @@ +locals { + # are we running in CI environment? + is_ci = can(get_env("CI")) + aws_account_id = trim(run_cmd("--terragrunt-quiet", "sh", "-c", "aws sts get-caller-identity --query=Account 2> /dev/null || true"), "\"") + aws_role_name = local.is_ci ? "TempGithubActionsRole" : try(get_env("TG_ROLE_NAME", "AodnTerraformAdminRole")) + aws_account = local.is_ci ? get_env("AWS_ACCOUNT_ID") : local.aws_account_id + aws_region = get_env("AWS_REGION", "ap-southeast-2") + state_bucket = "tfstate-${local.aws_account}-${local.aws_region}" + state_key = "apps/${basename(get_repo_root())}/${basename(get_terragrunt_dir())}.tfstate" + aws_role_arn = "arn:aws:iam::${local.aws_account}:role/${local.aws_role_name}" +} + +generate "providers" { + path = "providers.tf" + if_exists = "overwrite_terragrunt" + contents = < Date: Tue, 5 Dec 2023 19:13:36 +1100 Subject: [PATCH 04/55] update test workflow to include terragrunt deploy --- .github/workflows/test.yml | 28 ++++++++++++++++++++++++++++ 1 file changed, 28 insertions(+) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index b291403..498e2ca 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -26,11 +26,20 @@ jobs: - name: Set up Docker Buildx uses: docker/setup-buildx-action@v3 + - name: Expose environment + run: echo '${{ toJSON(github) }}' + - name: set tag + if: ${{ !env.ACT }} run: | BRANCH_NAME=${{ github.head_ref || github.ref_name }} echo TAG=${{ env.TAG_PREFIX}}-${BRANCH_NAME/\//_} >> $GITHUB_ENV + - name: set tag when running locally in act + if: ${{ env.ACT }} + run: | + echo TAG=latest >> $GITHUB_ENV + - name: Set up docker structure test run: > curl -LO @@ -39,6 +48,7 @@ jobs: /usr/local/bin/container-structure-test - name: Configure AWS Credentials + if: ${{ !env.ACT }} uses: aws-actions/configure-aws-credentials@v4 with: audience: sts.amazonaws.com @@ -46,6 +56,7 @@ jobs: role-to-assume: ${{ secrets.AWS_ROLE_ARN }} - name: Login to ECR + if: ${{ !env.ACT }} uses: docker/login-action@v3 with: registry: ${{ vars.ECR_REGISTRY }} @@ -62,9 +73,26 @@ jobs: container-structure-test test --image ${{ env.TAG }} --config tests/config.yaml - name: Build and push + if: ${{ !env.ACT }} uses: docker/build-push-action@v5 with: context: . platforms: linux/amd64,linux/arm64 push: true tags: ${{ vars.ECR_REGISTRY }}/${{ vars.ECR_REPOSITORY }}:${{ env.TAG }} + + staging_deploy_plan: + runs-on: ubuntu-latest + environment: staging + needs: build_test_push + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Gruntwork Terragrunt + uses: gruntwork-io/terragrunt-action@v2.0.0 + with: + tf_version: '1.5.7' + tg_version: '0.51.0' + tg_dir: './deploy/tg' + tg_command: 'run-all plan' From 1ec542be78576b197a71fae857ed25b7b6750391 Mon Sep 17 00:00:00 2001 From: Stefan Hattrell <29941279+digorgonzola@users.noreply.github.com> Date: Tue, 5 Dec 2023 19:21:22 +1100 Subject: [PATCH 05/55] test.yml: only build for amd64 --- .github/workflows/test.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 498e2ca..be1612c 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -77,7 +77,7 @@ jobs: uses: docker/build-push-action@v5 with: context: . - platforms: linux/amd64,linux/arm64 +# platforms: linux/amd64,linux/arm64 push: true tags: ${{ vars.ECR_REGISTRY }}/${{ vars.ECR_REPOSITORY }}:${{ env.TAG }} From c2bad01955ee60aa57aed403b58b82d3717cedcb Mon Sep 17 00:00:00 2001 From: Stefan Hattrell <29941279+digorgonzola@users.noreply.github.com> Date: Tue, 5 Dec 2023 19:28:56 +1100 Subject: [PATCH 06/55] test.yml: add debug information temp disable build --- .github/workflows/test.yml | 137 +++++++++++++++++++------------------ 1 file changed, 70 insertions(+), 67 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index be1612c..87d480c 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -14,81 +14,84 @@ permissions: contents: read # This is required for actions/checkout jobs: - build_test_push: - runs-on: ubuntu-latest - steps: - - name: Checkout - uses: actions/checkout@v4 - - - name: Set up QEMU - uses: docker/setup-qemu-action@v3 - - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v3 - - - name: Expose environment - run: echo '${{ toJSON(github) }}' - - - name: set tag - if: ${{ !env.ACT }} - run: | - BRANCH_NAME=${{ github.head_ref || github.ref_name }} - echo TAG=${{ env.TAG_PREFIX}}-${BRANCH_NAME/\//_} >> $GITHUB_ENV - - - name: set tag when running locally in act - if: ${{ env.ACT }} - run: | - echo TAG=latest >> $GITHUB_ENV - - - name: Set up docker structure test - run: > - curl -LO - https://storage.googleapis.com/container-structure-test/latest/container-structure-test-linux-amd64 - && chmod +x container-structure-test-linux-amd64 && sudo mv container-structure-test-linux-amd64 - /usr/local/bin/container-structure-test - - - name: Configure AWS Credentials - if: ${{ !env.ACT }} - uses: aws-actions/configure-aws-credentials@v4 - with: - audience: sts.amazonaws.com - aws-region: ${{ env.AWS_REGION }} - role-to-assume: ${{ secrets.AWS_ROLE_ARN }} - - - name: Login to ECR - if: ${{ !env.ACT }} - uses: docker/login-action@v3 - with: - registry: ${{ vars.ECR_REGISTRY }} - - - name: Build - uses: docker/build-push-action@v5 - with: - context: . - load: true - tags: ${{ env.TAG }} - - - name: Test - run: | - container-structure-test test --image ${{ env.TAG }} --config tests/config.yaml - - - name: Build and push - if: ${{ !env.ACT }} - uses: docker/build-push-action@v5 - with: - context: . -# platforms: linux/amd64,linux/arm64 - push: true - tags: ${{ vars.ECR_REGISTRY }}/${{ vars.ECR_REPOSITORY }}:${{ env.TAG }} +# build_test_push: +# runs-on: ubuntu-latest +# steps: +# - name: Checkout +# uses: actions/checkout@v4 +# +# - name: Set up QEMU +# uses: docker/setup-qemu-action@v3 +# +# - name: Set up Docker Buildx +# uses: docker/setup-buildx-action@v3 +# +# - name: Expose environment +# run: echo '${{ toJSON(github) }}' +# +# - name: set tag +# if: ${{ !env.ACT }} +# run: | +# BRANCH_NAME=${{ github.head_ref || github.ref_name }} +# echo TAG=${{ env.TAG_PREFIX}}-${BRANCH_NAME/\//_} >> $GITHUB_ENV +# +# - name: set tag when running locally in act +# if: ${{ env.ACT }} +# run: | +# echo TAG=latest >> $GITHUB_ENV +# +# - name: Set up docker structure test +# run: > +# curl -LO +# https://storage.googleapis.com/container-structure-test/latest/container-structure-test-linux-amd64 +# && chmod +x container-structure-test-linux-amd64 && sudo mv container-structure-test-linux-amd64 +# /usr/local/bin/container-structure-test +# +# - name: Configure AWS Credentials +# if: ${{ !env.ACT }} +# uses: aws-actions/configure-aws-credentials@v4 +# with: +# audience: sts.amazonaws.com +# aws-region: ${{ env.AWS_REGION }} +# role-to-assume: ${{ secrets.AWS_ROLE_ARN }} +# +# - name: Login to ECR +# if: ${{ !env.ACT }} +# uses: docker/login-action@v3 +# with: +# registry: ${{ vars.ECR_REGISTRY }} +# +# - name: Build +# uses: docker/build-push-action@v5 +# with: +# context: . +# load: true +# tags: ${{ env.TAG }} +# +# - name: Test +# run: | +# container-structure-test test --image ${{ env.TAG }} --config tests/config.yaml +# +# - name: Build and push +# if: ${{ !env.ACT }} +# uses: docker/build-push-action@v5 +# with: +# context: . +## platforms: linux/amd64,linux/arm64 +# push: true +# tags: ${{ vars.ECR_REGISTRY }}/${{ vars.ECR_REPOSITORY }}:${{ env.TAG }} staging_deploy_plan: runs-on: ubuntu-latest environment: staging - needs: build_test_push +# needs: build_test_push steps: - name: Checkout uses: actions/checkout@v4 + - name: Expose environment + run: echo '${{ toJSON(github) }}' + - name: Gruntwork Terragrunt uses: gruntwork-io/terragrunt-action@v2.0.0 with: From e3a4aa438da5878ef92709770f11cb0d11432e52 Mon Sep 17 00:00:00 2001 From: Stefan Hattrell <29941279+digorgonzola@users.noreply.github.com> Date: Tue, 5 Dec 2023 19:37:51 +1100 Subject: [PATCH 07/55] deploy: add .terraform.lock.hcl and correct order of inputs --- deploy/tf/ecs/.terraform.lock.hcl | 25 +++++++++++++++++++++++++ deploy/tg/ecs/terragrunt.hcl | 2 +- 2 files changed, 26 insertions(+), 1 deletion(-) create mode 100644 deploy/tf/ecs/.terraform.lock.hcl diff --git a/deploy/tf/ecs/.terraform.lock.hcl b/deploy/tf/ecs/.terraform.lock.hcl new file mode 100644 index 0000000..eabf303 --- /dev/null +++ b/deploy/tf/ecs/.terraform.lock.hcl @@ -0,0 +1,25 @@ +# This file is maintained automatically by "terraform init". +# Manual edits may be lost in future updates. + +provider "registry.terraform.io/hashicorp/aws" { + version = "5.29.0" + constraints = ">= 4.66.1" + hashes = [ + "h1:SyiKAX/D3ZE9My7P03DrRMf65pNnfSDQXPb0g11lCS0=", + "zh:0453c1c64e51cd7050ce46d9280a0195b9073592508077ebf1a1c45f7026f3f5", + "zh:3ee87d1a2870b61fdcc80f3f96b669dbcc8171aadb821bec0e1fa0e6fb9595b6", + "zh:423c0304eba345167cc37dcd300712f24f03fe4de8eecc15edb0d4f88b29ec79", + "zh:6816ce0ed702263297a8e02467bb712c509a9f6e4f132a152a10f1cc19191a81", + "zh:6feb8a0aedabd778216238e72273f5c2ee86d8841acc3fb3dc9d8014a2bbdc51", + "zh:709ccdc8b37f975d422e7955814671548887613931e234e06249da629b0f2f95", + "zh:76c55744020dbdafea25be634f8ac37c1e371f8c397f73bd89bc270d00ee0834", + "zh:7e48d6fc488b9dbe2fd4bebefa1b485d04da38b11a6799f8cba178173b7f8782", + "zh:951d7ef2adbfb96b1d3e9c4780b2ab4375caf9c6b522a2d023c02ff0698d8e2a", + "zh:9b12af85486a96aedd8d7984b0ff811a4b42e3d88dad1a3fb4c0b580d04fa425", + "zh:b0bf5974bc1a7d2ce3f3a9a31a8238ad15ad02211f1e84c54832541ec4bd5d10", + "zh:cc56d4ab9bcbee95f73dbe90f11d4ff7299b835dddf2b30cfda526a2cccd0f9f", + "zh:cfe3a4394f2f7044e03bb63f4fb9c691926607c6784417ac9c0724943da60d09", + "zh:d6f82e13f33f70de8df480287b5a961ced5606f041d1c589f706b112f68db890", + "zh:fb7be5bcff62d0ca9edd4a1bee4d2ed16e9428e3f9eff3ea4d898ecb234505a3", + ] +} diff --git a/deploy/tg/ecs/terragrunt.hcl b/deploy/tg/ecs/terragrunt.hcl index f102ae8..33ce4cc 100644 --- a/deploy/tg/ecs/terragrunt.hcl +++ b/deploy/tg/ecs/terragrunt.hcl @@ -19,8 +19,8 @@ include "global" { inputs = { app_hostnames = ["api", "sample-django-app"] db_secret_name = "/rds/stefan-db/primary/evaluation/api" - parameter_name = get_env("PARAMETER_NAME", "/apps/shared/devops/sydney") ecr_registry = get_env("ECR_REGISTRY", "450356697252.dkr.ecr.ap-southeast-2.amazonaws.com") + parameter_name = get_env("PARAMETER_NAME", "/apps/shared/devops/sydney") s3_buckets = dependency.s3.outputs.wrapper } From e33cd2de6cd7e65e4a3c2ea5b05cbb36b9eb0307 Mon Sep 17 00:00:00 2001 From: Stefan Hattrell <29941279+digorgonzola@users.noreply.github.com> Date: Tue, 5 Dec 2023 19:38:05 +1100 Subject: [PATCH 08/55] test.yml: include env vars --- .github/workflows/test.yml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 87d480c..ab3b72a 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -99,3 +99,8 @@ jobs: tg_version: '0.51.0' tg_dir: './deploy/tg' tg_command: 'run-all plan' + env: + AWS_ACCOUNT_ID: ${{ vars.AWS_ACCOUNT_ID }} + AWS_REGION: ${{ vars.AWS_REGION }} + ECR_REGISTRY: ${{ vars.ECR_REGISTRY }} + PARAMETER_NAME: ${{ vars.PARAMETER_NAME }} From b699b59852b8d25d9ff769cbdc0cb249577aa7a1 Mon Sep 17 00:00:00 2001 From: Stefan Hattrell <29941279+digorgonzola@users.noreply.github.com> Date: Tue, 5 Dec 2023 19:48:06 +1100 Subject: [PATCH 09/55] terragrunt.hcl: simplify aws config --- deploy/tg/terragrunt.hcl | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/deploy/tg/terragrunt.hcl b/deploy/tg/terragrunt.hcl index ef2f408..fe86c37 100644 --- a/deploy/tg/terragrunt.hcl +++ b/deploy/tg/terragrunt.hcl @@ -1,9 +1,7 @@ locals { - # are we running in CI environment? - is_ci = can(get_env("CI")) aws_account_id = trim(run_cmd("--terragrunt-quiet", "sh", "-c", "aws sts get-caller-identity --query=Account 2> /dev/null || true"), "\"") - aws_role_name = local.is_ci ? "TempGithubActionsRole" : try(get_env("TG_ROLE_NAME", "AodnTerraformAdminRole")) - aws_account = local.is_ci ? get_env("AWS_ACCOUNT_ID") : local.aws_account_id + aws_role_name = get_env("AWS_ROLE_NAME", "AodnTerraformAdminRole") + aws_account = get_env("AWS_ACCOUNT_ID", local.aws_account_id) aws_region = get_env("AWS_REGION", "ap-southeast-2") state_bucket = "tfstate-${local.aws_account}-${local.aws_region}" state_key = "apps/${basename(get_repo_root())}/${basename(get_terragrunt_dir())}.tfstate" From 457cc8f858fd0838301e8ae5bf371e86115c8ad1 Mon Sep 17 00:00:00 2001 From: Stefan Hattrell <29941279+digorgonzola@users.noreply.github.com> Date: Tue, 5 Dec 2023 19:48:21 +1100 Subject: [PATCH 10/55] test.yml: aws authentication --- .github/workflows/test.yml | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index ab3b72a..cf2e4c1 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -89,8 +89,13 @@ jobs: - name: Checkout uses: actions/checkout@v4 - - name: Expose environment - run: echo '${{ toJSON(github) }}' + - name: Configure AWS Credentials + if: ${{ !env.ACT }} + uses: aws-actions/configure-aws-credentials@v4 + with: + audience: sts.amazonaws.com + aws-region: ${{ vars.AWS_REGION }} + role-to-assume: arn:aws:iam::${{ vars.AWS_ACCOUNT_ID }}:role/${{ vars.AWS_ROLE_NAME }} - name: Gruntwork Terragrunt uses: gruntwork-io/terragrunt-action@v2.0.0 @@ -102,5 +107,6 @@ jobs: env: AWS_ACCOUNT_ID: ${{ vars.AWS_ACCOUNT_ID }} AWS_REGION: ${{ vars.AWS_REGION }} + AWS_ROLE_NAME: ${{ vars.AWS_ROLE_NAME }} ECR_REGISTRY: ${{ vars.ECR_REGISTRY }} PARAMETER_NAME: ${{ vars.PARAMETER_NAME }} From 5f7ef078ab8553823f848a7503cc1e456ea8a448 Mon Sep 17 00:00:00 2001 From: Stefan Hattrell <29941279+digorgonzola@users.noreply.github.com> Date: Tue, 5 Dec 2023 20:01:55 +1100 Subject: [PATCH 11/55] debug aws auth --- .github/workflows/test.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index cf2e4c1..b172d33 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -97,6 +97,9 @@ jobs: aws-region: ${{ vars.AWS_REGION }} role-to-assume: arn:aws:iam::${{ vars.AWS_ACCOUNT_ID }}:role/${{ vars.AWS_ROLE_NAME }} + - name: debug aws + run: aws sts get-caller-identity + - name: Gruntwork Terragrunt uses: gruntwork-io/terragrunt-action@v2.0.0 with: From e05ac30ce62a0dfc4fed786243a2acf3af1df51c Mon Sep 17 00:00:00 2001 From: Stefan Hattrell <29941279+digorgonzola@users.noreply.github.com> Date: Tue, 5 Dec 2023 20:05:40 +1100 Subject: [PATCH 12/55] simplify aws auth --- .github/workflows/test.yml | 4 ++-- deploy/tg/terragrunt.hcl | 20 ++++++++++---------- 2 files changed, 12 insertions(+), 12 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index b172d33..d2d6a9d 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -98,7 +98,7 @@ jobs: role-to-assume: arn:aws:iam::${{ vars.AWS_ACCOUNT_ID }}:role/${{ vars.AWS_ROLE_NAME }} - name: debug aws - run: aws sts get-caller-identity + run: aws sts get-caller-identity && aws s3 ls - name: Gruntwork Terragrunt uses: gruntwork-io/terragrunt-action@v2.0.0 @@ -110,6 +110,6 @@ jobs: env: AWS_ACCOUNT_ID: ${{ vars.AWS_ACCOUNT_ID }} AWS_REGION: ${{ vars.AWS_REGION }} - AWS_ROLE_NAME: ${{ vars.AWS_ROLE_NAME }} +# AWS_ROLE_NAME: ${{ vars.AWS_ROLE_NAME }} ECR_REGISTRY: ${{ vars.ECR_REGISTRY }} PARAMETER_NAME: ${{ vars.PARAMETER_NAME }} diff --git a/deploy/tg/terragrunt.hcl b/deploy/tg/terragrunt.hcl index fe86c37..27b26db 100644 --- a/deploy/tg/terragrunt.hcl +++ b/deploy/tg/terragrunt.hcl @@ -1,11 +1,11 @@ locals { aws_account_id = trim(run_cmd("--terragrunt-quiet", "sh", "-c", "aws sts get-caller-identity --query=Account 2> /dev/null || true"), "\"") - aws_role_name = get_env("AWS_ROLE_NAME", "AodnTerraformAdminRole") - aws_account = get_env("AWS_ACCOUNT_ID", local.aws_account_id) - aws_region = get_env("AWS_REGION", "ap-southeast-2") - state_bucket = "tfstate-${local.aws_account}-${local.aws_region}" - state_key = "apps/${basename(get_repo_root())}/${basename(get_terragrunt_dir())}.tfstate" - aws_role_arn = "arn:aws:iam::${local.aws_account}:role/${local.aws_role_name}" + # aws_role_name = get_env("AWS_ROLE_NAME", "AodnTerraformAdminRole") + aws_account = get_env("AWS_ACCOUNT_ID", local.aws_account_id) + aws_region = get_env("AWS_REGION", "ap-southeast-2") + state_bucket = "tfstate-${local.aws_account}-${local.aws_region}" + state_key = "apps/${basename(get_repo_root())}/${basename(get_terragrunt_dir())}.tfstate" + # aws_role_arn = "arn:aws:iam::${local.aws_account}:role/${local.aws_role_name}" } generate "providers" { @@ -15,9 +15,9 @@ generate "providers" { provider "aws" { region = "${local.aws_region}" allowed_account_ids = ["${local.aws_account}"] - assume_role { - role_arn = "${local.aws_role_arn}" - } +# assume_role { +# role_arn = "${local.aws_role_arn}" +# } default_tags { tags = { "Environment" = "apps" @@ -46,6 +46,6 @@ remote_state { skip_region_validation = true disable_bucket_update = true encrypt = true - role_arn = local.aws_role_arn + # role_arn = local.aws_role_arn } } From fb90b3b314d2064ad2844f84b041da4381426e9e Mon Sep 17 00:00:00 2001 From: Stefan Hattrell <29941279+digorgonzola@users.noreply.github.com> Date: Tue, 5 Dec 2023 20:07:24 +1100 Subject: [PATCH 13/55] fix provider config --- deploy/tg/terragrunt.hcl | 3 --- 1 file changed, 3 deletions(-) diff --git a/deploy/tg/terragrunt.hcl b/deploy/tg/terragrunt.hcl index 27b26db..b611776 100644 --- a/deploy/tg/terragrunt.hcl +++ b/deploy/tg/terragrunt.hcl @@ -15,9 +15,6 @@ generate "providers" { provider "aws" { region = "${local.aws_region}" allowed_account_ids = ["${local.aws_account}"] -# assume_role { -# role_arn = "${local.aws_role_arn}" -# } default_tags { tags = { "Environment" = "apps" From f3db34f191b7196912e959d72b92cf3ac97565c5 Mon Sep 17 00:00:00 2001 From: Stefan Hattrell <29941279+digorgonzola@users.noreply.github.com> Date: Tue, 5 Dec 2023 20:13:42 +1100 Subject: [PATCH 14/55] tg: remove further unnecessary bits --- deploy/tg/terragrunt.hcl | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/deploy/tg/terragrunt.hcl b/deploy/tg/terragrunt.hcl index b611776..b8f4f3f 100644 --- a/deploy/tg/terragrunt.hcl +++ b/deploy/tg/terragrunt.hcl @@ -1,11 +1,9 @@ locals { - aws_account_id = trim(run_cmd("--terragrunt-quiet", "sh", "-c", "aws sts get-caller-identity --query=Account 2> /dev/null || true"), "\"") - # aws_role_name = get_env("AWS_ROLE_NAME", "AodnTerraformAdminRole") aws_account = get_env("AWS_ACCOUNT_ID", local.aws_account_id) aws_region = get_env("AWS_REGION", "ap-southeast-2") + project_name = "sample-django-app" state_bucket = "tfstate-${local.aws_account}-${local.aws_region}" - state_key = "apps/${basename(get_repo_root())}/${basename(get_terragrunt_dir())}.tfstate" - # aws_role_arn = "arn:aws:iam::${local.aws_account}:role/${local.aws_role_name}" + state_key = "apps/${local.project_name}/${basename(get_terragrunt_dir())}.tfstate" } generate "providers" { @@ -43,6 +41,5 @@ remote_state { skip_region_validation = true disable_bucket_update = true encrypt = true - # role_arn = local.aws_role_arn } } From b7a69acc4aadf1a0df4eb3c4ecff269df1eb9862 Mon Sep 17 00:00:00 2001 From: Stefan Hattrell <29941279+digorgonzola@users.noreply.github.com> Date: Tue, 5 Dec 2023 20:13:53 +1100 Subject: [PATCH 15/55] test.yml: remove debug steps --- .github/workflows/test.yml | 4 ---- 1 file changed, 4 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index d2d6a9d..453a549 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -97,9 +97,6 @@ jobs: aws-region: ${{ vars.AWS_REGION }} role-to-assume: arn:aws:iam::${{ vars.AWS_ACCOUNT_ID }}:role/${{ vars.AWS_ROLE_NAME }} - - name: debug aws - run: aws sts get-caller-identity && aws s3 ls - - name: Gruntwork Terragrunt uses: gruntwork-io/terragrunt-action@v2.0.0 with: @@ -110,6 +107,5 @@ jobs: env: AWS_ACCOUNT_ID: ${{ vars.AWS_ACCOUNT_ID }} AWS_REGION: ${{ vars.AWS_REGION }} -# AWS_ROLE_NAME: ${{ vars.AWS_ROLE_NAME }} ECR_REGISTRY: ${{ vars.ECR_REGISTRY }} PARAMETER_NAME: ${{ vars.PARAMETER_NAME }} From 24a8b7b5f666c5c823fed0b239f825e99a56fde8 Mon Sep 17 00:00:00 2001 From: Stefan Hattrell <29941279+digorgonzola@users.noreply.github.com> Date: Tue, 5 Dec 2023 20:16:27 +1100 Subject: [PATCH 16/55] tg: fix aws account local --- deploy/tg/terragrunt.hcl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/deploy/tg/terragrunt.hcl b/deploy/tg/terragrunt.hcl index b8f4f3f..f29b281 100644 --- a/deploy/tg/terragrunt.hcl +++ b/deploy/tg/terragrunt.hcl @@ -1,5 +1,5 @@ locals { - aws_account = get_env("AWS_ACCOUNT_ID", local.aws_account_id) + aws_account = get_env("AWS_ACCOUNT_ID") aws_region = get_env("AWS_REGION", "ap-southeast-2") project_name = "sample-django-app" state_bucket = "tfstate-${local.aws_account}-${local.aws_region}" From c4f7fa7db92f56511b66309d702694b8d6f8009f Mon Sep 17 00:00:00 2001 From: Stefan Hattrell <29941279+digorgonzola@users.noreply.github.com> Date: Tue, 5 Dec 2023 20:30:26 +1100 Subject: [PATCH 17/55] add apply step --- .github/workflows/test.yml | 35 ++++++++++++++++++++++++++++++++++- 1 file changed, 34 insertions(+), 1 deletion(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 453a549..27c2879 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -97,7 +97,8 @@ jobs: aws-region: ${{ vars.AWS_REGION }} role-to-assume: arn:aws:iam::${{ vars.AWS_ACCOUNT_ID }}:role/${{ vars.AWS_ROLE_NAME }} - - name: Gruntwork Terragrunt + - name: Terragrunt Plan + id: terragrunt_plan uses: gruntwork-io/terragrunt-action@v2.0.0 with: tf_version: '1.5.7' @@ -109,3 +110,35 @@ jobs: AWS_REGION: ${{ vars.AWS_REGION }} ECR_REGISTRY: ${{ vars.ECR_REGISTRY }} PARAMETER_NAME: ${{ vars.PARAMETER_NAME }} + + staging_deploy_apply: + runs-on: ubuntu-latest + environment: staging + needs: staging_deploy_plan + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Configure AWS Credentials + if: ${{ !env.ACT }} + uses: aws-actions/configure-aws-credentials@v4 + with: + audience: sts.amazonaws.com + aws-region: ${{ vars.AWS_REGION }} + role-to-assume: arn:aws:iam::${{ vars.AWS_ACCOUNT_ID }}:role/${{ vars.AWS_ROLE_NAME }} + + - name: Terragrunt Plan + id: terragrunt_plan + uses: gruntwork-io/terragrunt-action@v2.0.0 + with: + tf_version: '1.5.7' + tg_version: '0.51.0' + tg_dir: './deploy/tg' + tg_command: 'run-all apply' + env: + AWS_ACCOUNT_ID: ${{ vars.AWS_ACCOUNT_ID }} + AWS_REGION: ${{ vars.AWS_REGION }} + ECR_REGISTRY: ${{ vars.ECR_REGISTRY }} + PARAMETER_NAME: ${{ vars.PARAMETER_NAME }} + TF_INPUT: 0 + TF_IN_AUTOMATION: true From e6ab59f68a890331be4da66cb5b35347597b8039 Mon Sep 17 00:00:00 2001 From: Stefan Hattrell <29941279+digorgonzola@users.noreply.github.com> Date: Tue, 5 Dec 2023 20:43:13 +1100 Subject: [PATCH 18/55] add image tag input --- deploy/tg/ecs/terragrunt.hcl | 1 + 1 file changed, 1 insertion(+) diff --git a/deploy/tg/ecs/terragrunt.hcl b/deploy/tg/ecs/terragrunt.hcl index 33ce4cc..aea6b73 100644 --- a/deploy/tg/ecs/terragrunt.hcl +++ b/deploy/tg/ecs/terragrunt.hcl @@ -20,6 +20,7 @@ inputs = { app_hostnames = ["api", "sample-django-app"] db_secret_name = "/rds/stefan-db/primary/evaluation/api" ecr_registry = get_env("ECR_REGISTRY", "450356697252.dkr.ecr.ap-southeast-2.amazonaws.com") + image_tag = get_env("IMAGE_TAG", "latest") parameter_name = get_env("PARAMETER_NAME", "/apps/shared/devops/sydney") s3_buckets = dependency.s3.outputs.wrapper } From e90e1a855d946db1b4b030cc82e56577b2da915d Mon Sep 17 00:00:00 2001 From: Stefan Hattrell <29941279+digorgonzola@users.noreply.github.com> Date: Tue, 5 Dec 2023 20:43:52 +1100 Subject: [PATCH 19/55] test.yml: pass image_tag to tg --- .github/workflows/test.yml | 143 +++++++++++++++++++------------------ deploy/tf/ecs/ecs.tf | 2 +- 2 files changed, 75 insertions(+), 70 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 27c2879..18ac2b5 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -14,77 +14,78 @@ permissions: contents: read # This is required for actions/checkout jobs: -# build_test_push: -# runs-on: ubuntu-latest -# steps: -# - name: Checkout -# uses: actions/checkout@v4 -# -# - name: Set up QEMU -# uses: docker/setup-qemu-action@v3 -# -# - name: Set up Docker Buildx -# uses: docker/setup-buildx-action@v3 -# -# - name: Expose environment -# run: echo '${{ toJSON(github) }}' -# -# - name: set tag -# if: ${{ !env.ACT }} -# run: | -# BRANCH_NAME=${{ github.head_ref || github.ref_name }} -# echo TAG=${{ env.TAG_PREFIX}}-${BRANCH_NAME/\//_} >> $GITHUB_ENV -# -# - name: set tag when running locally in act -# if: ${{ env.ACT }} -# run: | -# echo TAG=latest >> $GITHUB_ENV -# -# - name: Set up docker structure test -# run: > -# curl -LO -# https://storage.googleapis.com/container-structure-test/latest/container-structure-test-linux-amd64 -# && chmod +x container-structure-test-linux-amd64 && sudo mv container-structure-test-linux-amd64 -# /usr/local/bin/container-structure-test -# -# - name: Configure AWS Credentials -# if: ${{ !env.ACT }} -# uses: aws-actions/configure-aws-credentials@v4 -# with: -# audience: sts.amazonaws.com -# aws-region: ${{ env.AWS_REGION }} -# role-to-assume: ${{ secrets.AWS_ROLE_ARN }} -# -# - name: Login to ECR -# if: ${{ !env.ACT }} -# uses: docker/login-action@v3 -# with: -# registry: ${{ vars.ECR_REGISTRY }} -# -# - name: Build -# uses: docker/build-push-action@v5 -# with: -# context: . -# load: true -# tags: ${{ env.TAG }} -# -# - name: Test -# run: | -# container-structure-test test --image ${{ env.TAG }} --config tests/config.yaml -# -# - name: Build and push -# if: ${{ !env.ACT }} -# uses: docker/build-push-action@v5 -# with: -# context: . -## platforms: linux/amd64,linux/arm64 -# push: true -# tags: ${{ vars.ECR_REGISTRY }}/${{ vars.ECR_REPOSITORY }}:${{ env.TAG }} + build_test_push: + runs-on: ubuntu-latest + outputs: + image_tag: ${{ steps.set_tag.outputs.image_tag }} + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Set up QEMU + uses: docker/setup-qemu-action@v3 + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + + - name: set tag + id: set_tag + if: ${{ !env.ACT }} + run: | + BRANCH_NAME=${{ github.head_ref || github.ref_name }} + echo TAG=${{ env.TAG_PREFIX}}-${BRANCH_NAME/\//_} >> $GITHUB_ENV + echo "image_tag=${{ env.TAG_PREFIX}}-${BRANCH_NAME/\//_}" >> $GITHUB_OUTPUT + + - name: set tag when running locally in act + if: ${{ env.ACT }} + run: | + echo TAG=latest >> $GITHUB_ENV + + - name: Set up docker structure test + run: > + curl -LO + https://storage.googleapis.com/container-structure-test/latest/container-structure-test-linux-amd64 + && chmod +x container-structure-test-linux-amd64 && sudo mv container-structure-test-linux-amd64 + /usr/local/bin/container-structure-test + + - name: Configure AWS Credentials + if: ${{ !env.ACT }} + uses: aws-actions/configure-aws-credentials@v4 + with: + audience: sts.amazonaws.com + aws-region: ${{ env.AWS_REGION }} + role-to-assume: ${{ secrets.AWS_ROLE_ARN }} + + - name: Login to ECR + if: ${{ !env.ACT }} + uses: docker/login-action@v3 + with: + registry: ${{ vars.ECR_REGISTRY }} + + - name: Build + uses: docker/build-push-action@v5 + with: + context: . + load: true + tags: ${{ env.TAG }} + + - name: Test + run: | + container-structure-test test --image ${{ env.TAG }} --config tests/config.yaml + + - name: Build and push + if: ${{ !env.ACT }} + uses: docker/build-push-action@v5 + with: + context: . +# platforms: linux/amd64,linux/arm64 + push: true + tags: ${{ vars.ECR_REGISTRY }}/${{ vars.ECR_REPOSITORY }}:${{ env.TAG }} staging_deploy_plan: runs-on: ubuntu-latest environment: staging -# needs: build_test_push + needs: build_test_push steps: - name: Checkout uses: actions/checkout@v4 @@ -109,12 +110,15 @@ jobs: AWS_ACCOUNT_ID: ${{ vars.AWS_ACCOUNT_ID }} AWS_REGION: ${{ vars.AWS_REGION }} ECR_REGISTRY: ${{ vars.ECR_REGISTRY }} + IMAGE_TAG: ${{ needs.build_test_push.outputs.image_tag }} PARAMETER_NAME: ${{ vars.PARAMETER_NAME }} + TF_INPUT: 0 + TF_IN_AUTOMATION: true staging_deploy_apply: runs-on: ubuntu-latest environment: staging - needs: staging_deploy_plan + needs: [staging_deploy_plan, build_test_push] steps: - name: Checkout uses: actions/checkout@v4 @@ -127,7 +131,7 @@ jobs: aws-region: ${{ vars.AWS_REGION }} role-to-assume: arn:aws:iam::${{ vars.AWS_ACCOUNT_ID }}:role/${{ vars.AWS_ROLE_NAME }} - - name: Terragrunt Plan + - name: Terragrunt Apply id: terragrunt_plan uses: gruntwork-io/terragrunt-action@v2.0.0 with: @@ -139,6 +143,7 @@ jobs: AWS_ACCOUNT_ID: ${{ vars.AWS_ACCOUNT_ID }} AWS_REGION: ${{ vars.AWS_REGION }} ECR_REGISTRY: ${{ vars.ECR_REGISTRY }} + IMAGE_TAG: ${{ needs.build_test_push.outputs.image_tag }} PARAMETER_NAME: ${{ vars.PARAMETER_NAME }} TF_INPUT: 0 TF_IN_AUTOMATION: true diff --git a/deploy/tf/ecs/ecs.tf b/deploy/tf/ecs/ecs.tf index f61e660..11085f0 100644 --- a/deploy/tf/ecs/ecs.tf +++ b/deploy/tf/ecs/ecs.tf @@ -86,7 +86,7 @@ module "ecs" { } proxy = { name = "proxy" - image = "${var.ecr_registry}/nginx-proxy:${var.image_tag}" + image = "${var.ecr_registry}/nginx-proxy:latest" health_check = { command = ["CMD-SHELL", "curl -so /dev/null http://localhost/health || exit 1"] } From 5e20d4350705eda4032c3a7006bb88c1eb2a3f28 Mon Sep 17 00:00:00 2001 From: Stefan Hattrell <29941279+digorgonzola@users.noreply.github.com> Date: Tue, 5 Dec 2023 21:25:50 +1100 Subject: [PATCH 20/55] clean up docker test image + tag --- .github/workflows/test.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 18ac2b5..4bc795e 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -67,11 +67,11 @@ jobs: with: context: . load: true - tags: ${{ env.TAG }} + tags: ${{ vars.ECR_REPOSITORY }}:${{ env.TAG }} - name: Test run: | - container-structure-test test --image ${{ env.TAG }} --config tests/config.yaml + container-structure-test test --image ${{ vars.ECR_REPOSITORY }}:${{ env.TAG }} --config tests/config.yaml - name: Build and push if: ${{ !env.ACT }} From 82117f7da6c84f1af9f8cb530f29ac86cdd8a23c Mon Sep 17 00:00:00 2001 From: digorgonzola <29941279+digorgonzola@users.noreply.github.com> Date: Wed, 6 Dec 2023 12:42:44 +1100 Subject: [PATCH 21/55] remove obsolete tf files --- deploy/bastion.tf | 61 --------------------------------------------- deploy/locals.tf | 3 --- deploy/network.tf | 15 ----------- deploy/terraform.tf | 27 -------------------- 4 files changed, 106 deletions(-) delete mode 100644 deploy/bastion.tf delete mode 100644 deploy/locals.tf delete mode 100644 deploy/network.tf delete mode 100644 deploy/terraform.tf diff --git a/deploy/bastion.tf b/deploy/bastion.tf deleted file mode 100644 index af75cc8..0000000 --- a/deploy/bastion.tf +++ /dev/null @@ -1,61 +0,0 @@ -data "aws_ami" "amazon_linux" { - most_recent = true - filter { - name = "name" - values = ["amzn2-ami-hvm-2.0.*-x86_64-gp2"] - } - owners = ["amazon"] -} - -data "http" "myip" { - url = "https://wtfismyip.com/text" -} - -resource "tls_private_key" "bastion" { - rsa_bits = 4096 - algorithm = "ED25519" -} - -resource "local_file" "private_key" { - content = tls_private_key.bastion.private_key_openssh - filename = "id_${local.prefix}_bastion" - file_permission = "0600" -} - -resource "aws_key_pair" "bastion" { - public_key = tls_private_key.bastion.public_key_openssh - key_name = "${local.prefix}-bastion" -} - -resource "aws_instance" "bastion" { - ami = data.aws_ami.amazon_linux.id - instance_type = "t2.micro" - key_name = aws_key_pair.bastion.key_name - vpc_security_group_ids = [aws_security_group.bastion_host.id] - - tags = { - Name = "${local.prefix}-bastion" - } -} - -resource "aws_security_group" "bastion_host" { - name = "${local.prefix}-bastion" - description = "Bastion security group" - - ingress { - description = "Allow SSH in from specific IP" - from_port = 22 - to_port = 22 - protocol = "tcp" - - cidr_blocks = ["${trimspace(data.http.myip.response_body)}/32"] - } - - egress { - from_port = 0 - to_port = 0 - protocol = "-1" - description = "Allow bastion general internet access" - cidr_blocks = ["0.0.0.0/0"] - } -} diff --git a/deploy/locals.tf b/deploy/locals.tf deleted file mode 100644 index 23e4412..0000000 --- a/deploy/locals.tf +++ /dev/null @@ -1,3 +0,0 @@ -locals { - prefix = "stefan-${terraform.workspace}" -} diff --git a/deploy/network.tf b/deploy/network.tf deleted file mode 100644 index 01fa6b0..0000000 --- a/deploy/network.tf +++ /dev/null @@ -1,15 +0,0 @@ -resource "aws_vpc" "ecs_test" { - cidr_block = "10.1.0.0/16" - enable_dns_hostnames = true - enable_dns_support = true - tags = { - Name = "${local.prefix}-vpc" - } -} - -resource "aws_internet_gateway" "ecs_test" { - vpc_id = aws_vpc.ecs_test.id - tags = { - Name = "${local.prefix}-igw" - } -} diff --git a/deploy/terraform.tf b/deploy/terraform.tf deleted file mode 100644 index 1276560..0000000 --- a/deploy/terraform.tf +++ /dev/null @@ -1,27 +0,0 @@ -terraform { - required_version = "> 1.5.0" - required_providers { - aws = { - source = "hashicorp/aws" - version = "~> 5.0" - } - } - backend "s3" { - bucket = "tfstate-450356697252-ap-northeast-1" - key = "recipe-app.tfstate" - region = "ap-northeast-1" - encrypt = true - dynamodb_table = "tfstate-450356697252-ap-northeast-1" - } -} - -provider "aws" { - region = "ap-northeast-1" - allowed_account_ids = ["450356697252"] - default_tags { - tags = { - Owner = "Stefan" - Project = "ECS Training" - } - } -} From e26e164f197e85ae2fcffe0507d3b89d9d9dc614 Mon Sep 17 00:00:00 2001 From: digorgonzola <29941279+digorgonzola@users.noreply.github.com> Date: Wed, 6 Dec 2023 12:43:59 +1100 Subject: [PATCH 22/55] tg: add new module for ssm-parameter --- deploy/tg/ssm-parameter/terragrunt.hcl | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) create mode 100644 deploy/tg/ssm-parameter/terragrunt.hcl diff --git a/deploy/tg/ssm-parameter/terragrunt.hcl b/deploy/tg/ssm-parameter/terragrunt.hcl new file mode 100644 index 0000000..15acc09 --- /dev/null +++ b/deploy/tg/ssm-parameter/terragrunt.hcl @@ -0,0 +1,16 @@ +include "global" { + path = find_in_parent_folders("terragrunt.hcl") + expose = true +} + +inputs = { + parameter_name = get_env("PARAMETER_NAME", "/apps/shared/devops/sydney") +} + +locals { + global = include.global.locals +} + +terraform { + source = "${get_repo_root()}//deploy/tf/ssm-parameter" +} From 9396b04449880833226b7cacb43c04e1c64ec4a2 Mon Sep 17 00:00:00 2001 From: digorgonzola <29941279+digorgonzola@users.noreply.github.com> Date: Wed, 6 Dec 2023 12:44:18 +1100 Subject: [PATCH 23/55] tf: add new module for ssm-parameter --- deploy/tf/{ecs/parameters.tf => ssm-parameter/main.tf} | 0 deploy/tf/ssm-parameter/outputs.tf | 3 +++ deploy/tf/ssm-parameter/variables.tf | 3 +++ 3 files changed, 6 insertions(+) rename deploy/tf/{ecs/parameters.tf => ssm-parameter/main.tf} (100%) create mode 100644 deploy/tf/ssm-parameter/outputs.tf create mode 100644 deploy/tf/ssm-parameter/variables.tf diff --git a/deploy/tf/ecs/parameters.tf b/deploy/tf/ssm-parameter/main.tf similarity index 100% rename from deploy/tf/ecs/parameters.tf rename to deploy/tf/ssm-parameter/main.tf diff --git a/deploy/tf/ssm-parameter/outputs.tf b/deploy/tf/ssm-parameter/outputs.tf new file mode 100644 index 0000000..ae346cf --- /dev/null +++ b/deploy/tf/ssm-parameter/outputs.tf @@ -0,0 +1,3 @@ +output "parameter_values" { + value = nonsensitive(local.params) +} diff --git a/deploy/tf/ssm-parameter/variables.tf b/deploy/tf/ssm-parameter/variables.tf new file mode 100644 index 0000000..b8c86f6 --- /dev/null +++ b/deploy/tf/ssm-parameter/variables.tf @@ -0,0 +1,3 @@ +variable "parameter_name" { + type = string +} From 69399a4437c6699da6a4511152c8f59d07f2ed3e Mon Sep 17 00:00:00 2001 From: digorgonzola <29941279+digorgonzola@users.noreply.github.com> Date: Wed, 6 Dec 2023 12:44:40 +1100 Subject: [PATCH 24/55] tf/ecs: refactor ssm parameters to variables --- deploy/tf/ecs/alb.tf | 10 ++-- deploy/tf/ecs/ecs.tf | 38 +++++++------- deploy/tf/ecs/outputs.tf | 3 -- deploy/tf/ecs/variables.tf | 102 ++++++++++++++++++++++++++++++++----- 4 files changed, 114 insertions(+), 39 deletions(-) diff --git a/deploy/tf/ecs/alb.tf b/deploy/tf/ecs/alb.tf index b0f154a..6882f51 100644 --- a/deploy/tf/ecs/alb.tf +++ b/deploy/tf/ecs/alb.tf @@ -3,7 +3,7 @@ resource "aws_lb_target_group" "app" { port = 80 protocol = "HTTP" target_type = "ip" - vpc_id = local.params.vpc_id + vpc_id = var.vpc_id health_check { enabled = true @@ -13,20 +13,20 @@ resource "aws_lb_target_group" "app" { resource "aws_route53_record" "app" { for_each = toset(var.app_hostnames) - zone_id = local.params.dns_zone_id + zone_id = var.dns_zone_id name = each.value type = "A" alias { - name = local.params.alb_dns_name - zone_id = local.params.alb_zone_id + name = var.alb_dns_name + zone_id = var.alb_zone_id evaluate_target_health = true } } resource "aws_lb_listener_rule" "app_fgate" { for_each = toset(var.app_hostnames) - listener_arn = local.params.alb_https_listener + listener_arn = var.alb_listener_arn action { type = "forward" diff --git a/deploy/tf/ecs/ecs.tf b/deploy/tf/ecs/ecs.tf index 11085f0..2b6ee68 100644 --- a/deploy/tf/ecs/ecs.tf +++ b/deploy/tf/ecs/ecs.tf @@ -58,16 +58,16 @@ module "ecs" { essential = true memory_reservation = 256 environment = [ - { name = "DJANGO_SECRET_KEY", value = "changeme" }, - { name = "DB_HOST", value = local.params.rds_url }, - { name = "DB_NAME", value = "api" }, - { name = "DB_USER", value = "api" }, + { name = "ALLOWED_HOSTS", value = var.allowed_hosts }, + { name = "ALLOWED_CIDR_NETS", value = var.allowed_cidr_nets }, + { name = "DJANGO_SECRET_KEY", value = var.django_secret_key }, + { name = "DB_HOST", value = var.db_host }, + { name = "DB_NAME", value = var.db_name }, + { name = "DB_USER", value = var.db_user }, { name = "DB_SECRET_NAME", value = var.db_secret_name }, { name = "DB_SECRET_REGION", value = var.db_secret_region }, - { name = "ALLOWED_HOSTS", value = "*" }, - { name = "ALLOWED_CIDR_NETS", value = "${join(",", local.params.vpc_subnet_cidrs.private)}" }, - { name = "S3_STORAGE_BUCKET_NAME", value = var.s3_buckets["sample-django-app-bucket"].s3_bucket_id }, - { name = "S3_STORAGE_BUCKET_REGION", value = var.s3_buckets["sample-django-app-bucket"].s3_bucket_region } + { name = "S3_STORAGE_BUCKET_NAME", value = var.s3_storage_bucket_name }, + { name = "S3_STORAGE_BUCKET_REGION", value = var.s3_storage_bucket_region } ] port_mappings = [ { @@ -96,13 +96,13 @@ module "ecs" { environment = [ { name = "APP_HOST", value = "127.0.0.1" }, { name = "APP_PORT", value = 9000 }, - { name = "LISTEN_PORT", value = 80 } + { name = "LISTEN_PORT", value = var.container_port } ] port_mappings = [ { name = "nginx" - containerPort = 80 - hostPort = 80 + containerPort = var.container_port + hostPort = var.container_port } ] mount_points = [ @@ -124,19 +124,19 @@ module "ecs" { service = { target_group_arn = aws_lb_target_group.app.arn container_name = "proxy" - container_port = 80 + container_port = var.container_port } } - subnet_ids = local.params.vpc_private_subnets + subnet_ids = var.subnet_ids security_group_rules = { ingress_vpc = { type = "ingress" - from_port = 80 - to_port = 80 + from_port = var.container_port + to_port = var.container_port protocol = "tcp" - cidr_blocks = [local.params.vpc_cidr] + cidr_blocks = [var.vpc_cidr] } egress_all = { type = "egress" @@ -158,8 +158,8 @@ module "ecs" { "s3:PutObjectAcl" ] resources = [ - var.s3_buckets["sample-django-app-bucket"].s3_bucket_arn, - "${var.s3_buckets["sample-django-app-bucket"].s3_bucket_arn}/*" + "arn:aws:s3:::${var.s3_storage_bucket_name}", + "arn:aws:s3:::${var.s3_storage_bucket_name}/*" ] }, { @@ -167,7 +167,7 @@ module "ecs" { "secretsmanager:GetSecretValue" ] resources = [ - "arn:aws:secretsmanager:ap-southeast-2:450356697252:secret:/rds/stefan-db/primary/evaluation/api*" + "arn:aws:secretsmanager:${var.db_secret_region}:*:secret:${var.db_secret_name}*" ] } ] diff --git a/deploy/tf/ecs/outputs.tf b/deploy/tf/ecs/outputs.tf index b11c25e..e69de29 100644 --- a/deploy/tf/ecs/outputs.tf +++ b/deploy/tf/ecs/outputs.tf @@ -1,3 +0,0 @@ -#output "params" { -# value = nonsensitive(local.params) -#} diff --git a/deploy/tf/ecs/variables.tf b/deploy/tf/ecs/variables.tf index 78c7589..6567359 100644 --- a/deploy/tf/ecs/variables.tf +++ b/deploy/tf/ecs/variables.tf @@ -1,17 +1,32 @@ +variable "alb_dns_name" { + description = "The DNS name of the application load-balancer" + type = string +} + +variable "alb_zone_id" { + description = "The DNS zone ID of the application load-balancer" + type = string +} + +variable "alb_listener_arn" { + description = "The ARN of the application load-balancer listener" + type = string +} + variable "app_hostnames" { description = "Hostnames to associate with the application" type = list(string) } -variable "db_secret_name" { - description = "The name of the secret to fetch DB login credentials from" - type = string +variable "container_port" { + description = "The port to expose to the load balancer on the container" + type = number + default = 80 } -variable "db_secret_region" { - description = "The region to fetch the secret from" - type = string - default = "ap-southeast-2" +variable "dns_zone_id" { + description = "The ID of the route53 zone" + type = string } variable "ecr_registry" { @@ -19,17 +34,80 @@ variable "ecr_registry" { type = string } +variable "ecr_repository" { + description = "The name of the repository to pull the image from" + type = string +} + variable "image_tag" { description = "The tag of the docker image to pull from ECR" type = string default = "latest" } -variable "s3_buckets" { - description = "Map of s3 buckets" - type = any +variable "subnet_ids" { + description = "ID's of the subnets to deploy the container too" + type = list(string) } -variable "parameter_name" { - type = string +variable "vpc_cidr" { + description = "The CIDR of the VPC" + type = string +} + +variable "vpc_id" { + description = "The ID of the VPC" + type = string +} + +# Container environment variables +variable "allowed_hosts" { + description = "Hosts allowed to access the application container (i.e. 127.0.0.1)" + type = string +} + +variable "allowed_cidr_nets" { + description = "Subnet CIDR's allowed to access the application container" + type = string +} + +variable "django_secret_key" { + description = "The secret key for django app" + type = string +} + +variable "db_host" { + description = "The hostname of the database instance" + type = string +} + +variable "db_name" { + description = "The name of the database" + type = string +} + +variable "db_user" { + description = "The user to connect to the database with" + type = string +} + +variable "db_secret_name" { + description = "The name of the secret to fetch DB login credentials from" + type = string +} + +variable "db_secret_region" { + description = "The region to fetch the secret from" + type = string + default = "ap-southeast-2" +} + +variable "s3_storage_bucket_name" { + description = "Name of the S3 bucket to use" + type = string +} + +variable "s3_storage_bucket_region" { + description = "The bucket region" + type = string } From b9c65b9f41afa9801bcf037b9280e2a75301a3da Mon Sep 17 00:00:00 2001 From: digorgonzola <29941279+digorgonzola@users.noreply.github.com> Date: Wed, 6 Dec 2023 12:44:59 +1100 Subject: [PATCH 25/55] tg/ecs: update .terraform.lock.hcl --- deploy/tg/ecs/.terraform.lock.hcl | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/deploy/tg/ecs/.terraform.lock.hcl b/deploy/tg/ecs/.terraform.lock.hcl index cdc06e6..a648e46 100644 --- a/deploy/tg/ecs/.terraform.lock.hcl +++ b/deploy/tg/ecs/.terraform.lock.hcl @@ -2,8 +2,10 @@ # Manual edits may be lost in future updates. provider "registry.terraform.io/hashicorp/aws" { - version = "5.29.0" + version = "5.29.0" + constraints = ">= 4.66.1" hashes = [ + "h1:+4qYlyPaktjZTFP9UbpEaz55jfwWapdibViCPoRFf+s=", "h1:SyiKAX/D3ZE9My7P03DrRMf65pNnfSDQXPb0g11lCS0=", "zh:0453c1c64e51cd7050ce46d9280a0195b9073592508077ebf1a1c45f7026f3f5", "zh:3ee87d1a2870b61fdcc80f3f96b669dbcc8171aadb821bec0e1fa0e6fb9595b6", From 46d6c2f2b9eff049c1384108773830d35aaa4a98 Mon Sep 17 00:00:00 2001 From: digorgonzola <29941279+digorgonzola@users.noreply.github.com> Date: Wed, 6 Dec 2023 12:46:49 +1100 Subject: [PATCH 26/55] tg/ssm: add .terraform.lock.hcl --- deploy/tg/ssm-parameter/.terraform.lock.hcl | 24 +++++++++++++++++++++ 1 file changed, 24 insertions(+) create mode 100644 deploy/tg/ssm-parameter/.terraform.lock.hcl diff --git a/deploy/tg/ssm-parameter/.terraform.lock.hcl b/deploy/tg/ssm-parameter/.terraform.lock.hcl new file mode 100644 index 0000000..47d4158 --- /dev/null +++ b/deploy/tg/ssm-parameter/.terraform.lock.hcl @@ -0,0 +1,24 @@ +# This file is maintained automatically by "terraform init". +# Manual edits may be lost in future updates. + +provider "registry.terraform.io/hashicorp/aws" { + version = "5.29.0" + hashes = [ + "h1:+4qYlyPaktjZTFP9UbpEaz55jfwWapdibViCPoRFf+s=", + "zh:0453c1c64e51cd7050ce46d9280a0195b9073592508077ebf1a1c45f7026f3f5", + "zh:3ee87d1a2870b61fdcc80f3f96b669dbcc8171aadb821bec0e1fa0e6fb9595b6", + "zh:423c0304eba345167cc37dcd300712f24f03fe4de8eecc15edb0d4f88b29ec79", + "zh:6816ce0ed702263297a8e02467bb712c509a9f6e4f132a152a10f1cc19191a81", + "zh:6feb8a0aedabd778216238e72273f5c2ee86d8841acc3fb3dc9d8014a2bbdc51", + "zh:709ccdc8b37f975d422e7955814671548887613931e234e06249da629b0f2f95", + "zh:76c55744020dbdafea25be634f8ac37c1e371f8c397f73bd89bc270d00ee0834", + "zh:7e48d6fc488b9dbe2fd4bebefa1b485d04da38b11a6799f8cba178173b7f8782", + "zh:951d7ef2adbfb96b1d3e9c4780b2ab4375caf9c6b522a2d023c02ff0698d8e2a", + "zh:9b12af85486a96aedd8d7984b0ff811a4b42e3d88dad1a3fb4c0b580d04fa425", + "zh:b0bf5974bc1a7d2ce3f3a9a31a8238ad15ad02211f1e84c54832541ec4bd5d10", + "zh:cc56d4ab9bcbee95f73dbe90f11d4ff7299b835dddf2b30cfda526a2cccd0f9f", + "zh:cfe3a4394f2f7044e03bb63f4fb9c691926607c6784417ac9c0724943da60d09", + "zh:d6f82e13f33f70de8df480287b5a961ced5606f041d1c589f706b112f68db890", + "zh:fb7be5bcff62d0ca9edd4a1bee4d2ed16e9428e3f9eff3ea4d898ecb234505a3", + ] +} From 873f8cdfb1c49b29274381eba0a06321db384396 Mon Sep 17 00:00:00 2001 From: digorgonzola <29941279+digorgonzola@users.noreply.github.com> Date: Wed, 6 Dec 2023 12:47:15 +1100 Subject: [PATCH 27/55] tg/ecs: refactor use of params into vars ssm parameters will be passed in via another tg module --- deploy/tg/ecs/terragrunt.hcl | 59 +++++++++++++++++++++++++++++++++--- 1 file changed, 55 insertions(+), 4 deletions(-) diff --git a/deploy/tg/ecs/terragrunt.hcl b/deploy/tg/ecs/terragrunt.hcl index aea6b73..f97abcd 100644 --- a/deploy/tg/ecs/terragrunt.hcl +++ b/deploy/tg/ecs/terragrunt.hcl @@ -1,3 +1,32 @@ +dependency "ssm" { + config_path = "../ssm-parameter" + mock_outputs = { + parameter_values = { + "alb_arn" = "arn:aws:elasticloadbalancing:ap-southeast-2:450356697252:loadbalancer/app/shared-alb-devops-sydney/45b0c41ea845014b" + "alb_dns_name" = "shared-alb-devops-sydney-387767645.ap-southeast-2.elb.amazonaws.com" + "alb_https_listener" = "arn:aws:elasticloadbalancing:ap-southeast-2:450356697252:listener/app/shared-alb-devops-sydney/45b0c41ea845014b/ecf16c7cec52b0c9" + "alb_zone_id" = "Z1GM3OXH4ZPM65" + "dns_domain_name" = "gamma.aodn.org.au" + "dns_zone_id" = "Z03033261P68C2JNNSWPQ" + "rds_url" = "stefan-db-rds-primary-evaluation.gamma.aodn.org.au" + "vpc_cidr" = "10.32.0.0/16" + "vpc_id" = "vpc-006dcec5d49e40003" + "vpc_private_subnets" = [ + "subnet-06665685d5a875465", + "subnet-0df67c37ed85ed7f2", + "subnet-075ea5fbe179b2b9a", + ] + "vpc_subnet_cidrs" = { + "private" = [ + "10.32.48.0/20", + "10.32.64.0/20", + "10.32.80.0/20", + ] + } + } + } +} + dependency "s3" { config_path = "../s3" mock_outputs = { @@ -17,12 +46,34 @@ include "global" { } inputs = { - app_hostnames = ["api", "sample-django-app"] - db_secret_name = "/rds/stefan-db/primary/evaluation/api" + # DNS hostnames to associate with the container + app_hostnames = ["api", "sample-django-app"] + + # container image repository and tag ecr_registry = get_env("ECR_REGISTRY", "450356697252.dkr.ecr.ap-southeast-2.amazonaws.com") + ecr_repository = get_env("ECR_REPOSITORY", "api") image_tag = get_env("IMAGE_TAG", "latest") - parameter_name = get_env("PARAMETER_NAME", "/apps/shared/devops/sydney") - s3_buckets = dependency.s3.outputs.wrapper + + # Shared infrastructure details + alb_dns_name = dependency.ssm.outputs.parameter_values.alb_dns_name + alb_listener_arn = dependency.ssm.outputs.parameter_values.alb_https_listener + alb_zone_id = dependency.ssm.outputs.parameter_values.alb_zone_id + dns_zone_id = dependency.ssm.outputs.parameter_values.dns_zone_id + subnet_ids = dependency.ssm.outputs.parameter_values.vpc_private_subnets + vpc_cidr = dependency.ssm.outputs.parameter_values.vpc_cidr + vpc_id = dependency.ssm.outputs.parameter_values.vpc_id + + # get docker environment variable values with default fallback values + allowed_hosts = get_env("ALLOWED_HOSTS", "*") + allowed_cidr_nets = get_env("ALLOWED_CIDR_NETS", join(",", dependency.ssm.outputs.parameter_values.vpc_subnet_cidrs.private)) + django_secret_key = get_env("DJANGO_SECRET_KEY", "changeme") + db_host = get_env("DB_HOST", dependency.ssm.outputs.parameter_values.rds_url) + db_name = get_env("DB_NAME", "api") + db_user = get_env("DB_USER", "api") + db_secret_name = get_env("DB_SECRET_NAME", "/rds/stefan-db/primary/evaluation/api") + db_secret_region = get_env("DB_SECRET_REGION", "ap-southeast-2") + s3_storage_bucket_name = get_env("S3_STORAGE_BUCKET_NAME", dependency.s3.outputs.wrapper["sample-django-app-bucket"].s3_bucket_id) + s3_storage_bucket_region = get_env("S3_STORAGE_BUCKET_REGION", dependency.s3.outputs.wrapper["sample-django-app-bucket"].s3_bucket_region) } locals { From f04eab58915e12d772a99848261c5bf450ebc5b7 Mon Sep 17 00:00:00 2001 From: digorgonzola <29941279+digorgonzola@users.noreply.github.com> Date: Wed, 6 Dec 2023 12:48:19 +1100 Subject: [PATCH 28/55] update .gitignore --- .gitignore | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.gitignore b/.gitignore index 266a32f..1ab3559 100644 --- a/.gitignore +++ b/.gitignore @@ -106,7 +106,8 @@ venv.bak/ # IDE related config .idea/ -# Terraform directories and files +# Terraform/Terragrunt directories and files **/.terraform/* +**/.terragrunt-cache/ *.tfstate *.tfstate.* From ec97807f4fae6fec67eb12ad7772947041d77277 Mon Sep 17 00:00:00 2001 From: digorgonzola <29941279+digorgonzola@users.noreply.github.com> Date: Wed, 6 Dec 2023 23:13:36 +1100 Subject: [PATCH 29/55] test.yml: expose all vars/secrets to env context --- .github/workflows/test.yml | 36 ++++++++++++++++++++++++++---------- 1 file changed, 26 insertions(+), 10 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 4bc795e..68cacbf 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -2,6 +2,8 @@ name: build and test on: pull_request: + types: + - opened branches: - "master" @@ -98,6 +100,18 @@ jobs: aws-region: ${{ vars.AWS_REGION }} role-to-assume: arn:aws:iam::${{ vars.AWS_ACCOUNT_ID }}:role/${{ vars.AWS_ROLE_NAME }} + - name: Expose github environment as shell variables + env: + SECRETS_CONTEXT: ${{ toJson(secrets) }} + VARS_CONTEXT: ${{ toJson(vars) }} + run: | + # https://docs.github.com/en/actions/using-workflows/workflow-commands-for-github-actions#setting-an-environment-variable + # https://docs.github.com/en/actions/using-workflows/workflow-commands-for-github-actions#multiline-strings + EOF=$(dd if=/dev/urandom bs=15 count=1 status=none | base64) + to_envs() { jq -r "to_entries[] | \"\(.key)<<$EOF\n\(.value)\n$EOF\n\""; } + echo "$VARS_CONTEXT" | to_envs >> $GITHUB_ENV + echo "$SECRETS_CONTEXT" | to_envs >> $GITHUB_ENV + - name: Terragrunt Plan id: terragrunt_plan uses: gruntwork-io/terragrunt-action@v2.0.0 @@ -107,11 +121,6 @@ jobs: tg_dir: './deploy/tg' tg_command: 'run-all plan' env: - AWS_ACCOUNT_ID: ${{ vars.AWS_ACCOUNT_ID }} - AWS_REGION: ${{ vars.AWS_REGION }} - ECR_REGISTRY: ${{ vars.ECR_REGISTRY }} - IMAGE_TAG: ${{ needs.build_test_push.outputs.image_tag }} - PARAMETER_NAME: ${{ vars.PARAMETER_NAME }} TF_INPUT: 0 TF_IN_AUTOMATION: true @@ -131,6 +140,18 @@ jobs: aws-region: ${{ vars.AWS_REGION }} role-to-assume: arn:aws:iam::${{ vars.AWS_ACCOUNT_ID }}:role/${{ vars.AWS_ROLE_NAME }} + - name: Expose github environment as shell variables + env: + SECRETS_CONTEXT: ${{ toJson(secrets) }} + VARS_CONTEXT: ${{ toJson(vars) }} + run: | + # https://docs.github.com/en/actions/using-workflows/workflow-commands-for-github-actions#setting-an-environment-variable + # https://docs.github.com/en/actions/using-workflows/workflow-commands-for-github-actions#multiline-strings + EOF=$(dd if=/dev/urandom bs=15 count=1 status=none | base64) + to_envs() { jq -r "to_entries[] | \"\(.key)<<$EOF\n\(.value)\n$EOF\n\""; } + echo "$VARS_CONTEXT" | to_envs >> $GITHUB_ENV + echo "$SECRETS_CONTEXT" | to_envs >> $GITHUB_ENV + - name: Terragrunt Apply id: terragrunt_plan uses: gruntwork-io/terragrunt-action@v2.0.0 @@ -140,10 +161,5 @@ jobs: tg_dir: './deploy/tg' tg_command: 'run-all apply' env: - AWS_ACCOUNT_ID: ${{ vars.AWS_ACCOUNT_ID }} - AWS_REGION: ${{ vars.AWS_REGION }} - ECR_REGISTRY: ${{ vars.ECR_REGISTRY }} - IMAGE_TAG: ${{ needs.build_test_push.outputs.image_tag }} - PARAMETER_NAME: ${{ vars.PARAMETER_NAME }} TF_INPUT: 0 TF_IN_AUTOMATION: true From 60b547ee989b88aec156df5e62808eab16058973 Mon Sep 17 00:00:00 2001 From: digorgonzola <29941279+digorgonzola@users.noreply.github.com> Date: Wed, 6 Dec 2023 23:14:01 +1100 Subject: [PATCH 30/55] tg/ssm: rename env var --- deploy/tg/ssm-parameter/terragrunt.hcl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/deploy/tg/ssm-parameter/terragrunt.hcl b/deploy/tg/ssm-parameter/terragrunt.hcl index 15acc09..731f501 100644 --- a/deploy/tg/ssm-parameter/terragrunt.hcl +++ b/deploy/tg/ssm-parameter/terragrunt.hcl @@ -4,7 +4,7 @@ include "global" { } inputs = { - parameter_name = get_env("PARAMETER_NAME", "/apps/shared/devops/sydney") + parameter_name = get_env("SSM_PARAMETER_NAME", "/apps/shared/devops/sydney") } locals { From 7a189ffb932316a89ade1ae18090c9e8cc5b6d75 Mon Sep 17 00:00:00 2001 From: digorgonzola <29941279+digorgonzola@users.noreply.github.com> Date: Wed, 6 Dec 2023 23:17:48 +1100 Subject: [PATCH 31/55] update test.yml on event types --- .github/workflows/test.yml | 2 -- 1 file changed, 2 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 68cacbf..a234a49 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -2,8 +2,6 @@ name: build and test on: pull_request: - types: - - opened branches: - "master" From 470656651dea8548a04d03fc7524746ab5101578 Mon Sep 17 00:00:00 2001 From: digorgonzola <29941279+digorgonzola@users.noreply.github.com> Date: Wed, 6 Dec 2023 23:48:40 +1100 Subject: [PATCH 32/55] deploy: refactor to include environment name --- deploy/tf/ecs/ecs.tf | 4 ++-- deploy/tf/ecs/variables.tf | 5 +++++ deploy/tg/ecs/terragrunt.hcl | 14 +++++++++----- deploy/tg/s3/.terraform.lock.hcl | 4 +++- deploy/tg/s3/terragrunt.hcl | 4 ++-- deploy/tg/terragrunt.hcl | 3 ++- 6 files changed, 23 insertions(+), 11 deletions(-) diff --git a/deploy/tf/ecs/ecs.tf b/deploy/tf/ecs/ecs.tf index 2b6ee68..121d1e6 100644 --- a/deploy/tf/ecs/ecs.tf +++ b/deploy/tf/ecs/ecs.tf @@ -3,7 +3,7 @@ module "ecs" { version = "~> 5.7.0" # Cluster Configuration - cluster_name = "sample-django-app" + cluster_name = "sample-django-app-${var.environment}" cluster_configuration = { name = "containerInsights" value = "enabled" @@ -25,7 +25,7 @@ module "ecs" { # Service Configuration services = { - sample-django-app = { + "sample-django-app-${var.environment}" = { capacity_provider_strategy = { dedicated = { base = 0 diff --git a/deploy/tf/ecs/variables.tf b/deploy/tf/ecs/variables.tf index 6567359..0c124c5 100644 --- a/deploy/tf/ecs/variables.tf +++ b/deploy/tf/ecs/variables.tf @@ -45,6 +45,11 @@ variable "image_tag" { default = "latest" } +variable "environment" { + description = "Environment name to prepend/append to resource names" + type = string +} + variable "subnet_ids" { description = "ID's of the subnets to deploy the container too" type = list(string) diff --git a/deploy/tg/ecs/terragrunt.hcl b/deploy/tg/ecs/terragrunt.hcl index f97abcd..866e3f4 100644 --- a/deploy/tg/ecs/terragrunt.hcl +++ b/deploy/tg/ecs/terragrunt.hcl @@ -31,9 +31,9 @@ dependency "s3" { config_path = "../s3" mock_outputs = { wrapper = { - sample-django-app-bucket = { - s3_bucket_arn = "arn:aws:s3:::sample-django-app-123456789012" - s3_bucket_id = "sample-django-app-123456789012" + "sample-django-app-${local.global.environment}-${local.global.aws_account}" = { + s3_bucket_arn = "arn:aws:s3:::sample-django-app-${local.global.environment}-${local.global.aws_account}" + s3_bucket_id = "sample-django-app-${local.global.environment}-${local.global.aws_account}" s3_bucket_region = "ap-southeast-2" } } @@ -46,6 +46,8 @@ include "global" { } inputs = { + environment = local.global.environment + # DNS hostnames to associate with the container app_hostnames = ["api", "sample-django-app"] @@ -72,8 +74,10 @@ inputs = { db_user = get_env("DB_USER", "api") db_secret_name = get_env("DB_SECRET_NAME", "/rds/stefan-db/primary/evaluation/api") db_secret_region = get_env("DB_SECRET_REGION", "ap-southeast-2") - s3_storage_bucket_name = get_env("S3_STORAGE_BUCKET_NAME", dependency.s3.outputs.wrapper["sample-django-app-bucket"].s3_bucket_id) - s3_storage_bucket_region = get_env("S3_STORAGE_BUCKET_REGION", dependency.s3.outputs.wrapper["sample-django-app-bucket"].s3_bucket_region) + s3_storage_bucket_name = get_env("S3_STORAGE_BUCKET_NAME", + dependency.s3.outputs.wrapper["sample-django-app-${local.global.environment}-${local.global.aws_account}"].s3_bucket_id) + s3_storage_bucket_region = get_env("S3_STORAGE_BUCKET_REGION", + dependency.s3.outputs.wrapper["sample-django-app-${local.global.environment}-${local.global.aws_account}"].s3_bucket_region) } locals { diff --git a/deploy/tg/s3/.terraform.lock.hcl b/deploy/tg/s3/.terraform.lock.hcl index cdc06e6..700fcab 100644 --- a/deploy/tg/s3/.terraform.lock.hcl +++ b/deploy/tg/s3/.terraform.lock.hcl @@ -2,8 +2,10 @@ # Manual edits may be lost in future updates. provider "registry.terraform.io/hashicorp/aws" { - version = "5.29.0" + version = "5.29.0" + constraints = ">= 4.9.0" hashes = [ + "h1:+4qYlyPaktjZTFP9UbpEaz55jfwWapdibViCPoRFf+s=", "h1:SyiKAX/D3ZE9My7P03DrRMf65pNnfSDQXPb0g11lCS0=", "zh:0453c1c64e51cd7050ce46d9280a0195b9073592508077ebf1a1c45f7026f3f5", "zh:3ee87d1a2870b61fdcc80f3f96b669dbcc8171aadb821bec0e1fa0e6fb9595b6", diff --git a/deploy/tg/s3/terragrunt.hcl b/deploy/tg/s3/terragrunt.hcl index 51a6d79..011ac0b 100644 --- a/deploy/tg/s3/terragrunt.hcl +++ b/deploy/tg/s3/terragrunt.hcl @@ -5,8 +5,8 @@ include "global" { inputs = { items = { - sample-django-app-bucket = { - bucket = "sample-django-app-${local.global.aws_account}" + "sample-django-app-${local.global.environment}-${local.global.aws_account}" = { + bucket = "sample-django-app-${local.global.environment}-${local.global.aws_account}" acl = "public-read" block_public_acls = false block_public_policy = false diff --git a/deploy/tg/terragrunt.hcl b/deploy/tg/terragrunt.hcl index f29b281..b2b13d4 100644 --- a/deploy/tg/terragrunt.hcl +++ b/deploy/tg/terragrunt.hcl @@ -1,9 +1,10 @@ locals { aws_account = get_env("AWS_ACCOUNT_ID") aws_region = get_env("AWS_REGION", "ap-southeast-2") + environment = get_env("ENVIRONMENT") project_name = "sample-django-app" state_bucket = "tfstate-${local.aws_account}-${local.aws_region}" - state_key = "apps/${local.project_name}/${basename(get_terragrunt_dir())}.tfstate" + state_key = "apps/${local.project_name}/${local.environment}/${basename(get_terragrunt_dir())}.tfstate" } generate "providers" { From 0acde6caa5f9f9edd1530bf76a11328724c5b8c3 Mon Sep 17 00:00:00 2001 From: digorgonzola <29941279+digorgonzola@users.noreply.github.com> Date: Wed, 6 Dec 2023 23:53:40 +1100 Subject: [PATCH 33/55] test.yml: set docker build job environment --- .github/workflows/test.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index a234a49..81221a4 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -16,6 +16,7 @@ permissions: jobs: build_test_push: runs-on: ubuntu-latest + environment: staging outputs: image_tag: ${{ steps.set_tag.outputs.image_tag }} steps: From cfb26f03a5e18451edb150c94128557465f9e423 Mon Sep 17 00:00:00 2001 From: digorgonzola <29941279+digorgonzola@users.noreply.github.com> Date: Wed, 6 Dec 2023 23:59:32 +1100 Subject: [PATCH 34/55] test.yml: update aws credential steps --- .github/workflows/test.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 81221a4..9b6fde2 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -97,7 +97,7 @@ jobs: with: audience: sts.amazonaws.com aws-region: ${{ vars.AWS_REGION }} - role-to-assume: arn:aws:iam::${{ vars.AWS_ACCOUNT_ID }}:role/${{ vars.AWS_ROLE_NAME }} + role-to-assume: ${{ secrets.AWS_ROLE_ARN }} - name: Expose github environment as shell variables env: @@ -137,7 +137,7 @@ jobs: with: audience: sts.amazonaws.com aws-region: ${{ vars.AWS_REGION }} - role-to-assume: arn:aws:iam::${{ vars.AWS_ACCOUNT_ID }}:role/${{ vars.AWS_ROLE_NAME }} + role-to-assume: ${{ secrets.AWS_ROLE_ARN }} - name: Expose github environment as shell variables env: From 551b58569f9ea75eae54eb9312d45ad4b5e8bb87 Mon Sep 17 00:00:00 2001 From: digorgonzola <29941279+digorgonzola@users.noreply.github.com> Date: Thu, 7 Dec 2023 00:09:24 +1100 Subject: [PATCH 35/55] tg/ecs: update route53 records --- deploy/tg/ecs/terragrunt.hcl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/deploy/tg/ecs/terragrunt.hcl b/deploy/tg/ecs/terragrunt.hcl index 866e3f4..37742c6 100644 --- a/deploy/tg/ecs/terragrunt.hcl +++ b/deploy/tg/ecs/terragrunt.hcl @@ -49,7 +49,7 @@ inputs = { environment = local.global.environment # DNS hostnames to associate with the container - app_hostnames = ["api", "sample-django-app"] + app_hostnames = ["api-${local.global.environment}"] # container image repository and tag ecr_registry = get_env("ECR_REGISTRY", "450356697252.dkr.ecr.ap-southeast-2.amazonaws.com") From b52a198a90f6a59c1be28f6e6e9f7ad2e0e3ad62 Mon Sep 17 00:00:00 2001 From: Stefan Hattrell <29941279+digorgonzola@users.noreply.github.com> Date: Fri, 8 Dec 2023 11:43:31 +1100 Subject: [PATCH 36/55] test.yml: expose ssm param values via tfvars.json file --- .github/workflows/test.yml | 76 +++++++++++++++++++++++++------------- 1 file changed, 51 insertions(+), 25 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 9b6fde2..01b9d37 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -1,4 +1,4 @@ -name: build and test +name: Build, Test and Deploy on: pull_request: @@ -7,7 +7,6 @@ on: env: AWS_REGION: ap-southeast-2 - TAG_PREFIX: test permissions: id-token: write # This is required for requesting the JWT @@ -18,7 +17,7 @@ jobs: runs-on: ubuntu-latest environment: staging outputs: - image_tag: ${{ steps.set_tag.outputs.image_tag }} + image_digest: ${{ steps.build_and_push.outputs.digest }} steps: - name: Checkout uses: actions/checkout@v4 @@ -29,20 +28,16 @@ jobs: - name: Set up Docker Buildx uses: docker/setup-buildx-action@v3 - - name: set tag + - name: Set Image Tag id: set_tag - if: ${{ !env.ACT }} run: | BRANCH_NAME=${{ github.head_ref || github.ref_name }} - echo TAG=${{ env.TAG_PREFIX}}-${BRANCH_NAME/\//_} >> $GITHUB_ENV + echo TAG=${{ env.TAG_PREFIX }}-${BRANCH_NAME//\//_} >> $GITHUB_ENV echo "image_tag=${{ env.TAG_PREFIX}}-${BRANCH_NAME/\//_}" >> $GITHUB_OUTPUT + env: + TAG_PREFIX: dev - - name: set tag when running locally in act - if: ${{ env.ACT }} - run: | - echo TAG=latest >> $GITHUB_ENV - - - name: Set up docker structure test + - name: Setup Docker Structure Test run: > curl -LO https://storage.googleapis.com/container-structure-test/latest/container-structure-test-linux-amd64 @@ -50,7 +45,6 @@ jobs: /usr/local/bin/container-structure-test - name: Configure AWS Credentials - if: ${{ !env.ACT }} uses: aws-actions/configure-aws-credentials@v4 with: audience: sts.amazonaws.com @@ -58,27 +52,27 @@ jobs: role-to-assume: ${{ secrets.AWS_ROLE_ARN }} - name: Login to ECR - if: ${{ !env.ACT }} uses: docker/login-action@v3 with: registry: ${{ vars.ECR_REGISTRY }} - - name: Build + - name: Build Docker Image uses: docker/build-push-action@v5 with: context: . load: true tags: ${{ vars.ECR_REPOSITORY }}:${{ env.TAG }} - - name: Test + - name: Test Docker Image run: | container-structure-test test --image ${{ vars.ECR_REPOSITORY }}:${{ env.TAG }} --config tests/config.yaml - - name: Build and push - if: ${{ !env.ACT }} + - name: Build and Push Docker Image + id: build_and_push uses: docker/build-push-action@v5 with: context: . +# Only building for AMD64 for now # platforms: linux/amd64,linux/arm64 push: true tags: ${{ vars.ECR_REGISTRY }}/${{ vars.ECR_REPOSITORY }}:${{ env.TAG }} @@ -92,20 +86,34 @@ jobs: uses: actions/checkout@v4 - name: Configure AWS Credentials - if: ${{ !env.ACT }} uses: aws-actions/configure-aws-credentials@v4 with: audience: sts.amazonaws.com aws-region: ${{ vars.AWS_REGION }} role-to-assume: ${{ secrets.AWS_ROLE_ARN }} + - name: Export shared infrastructure SSM parameter values to auto.tfvars.json files + env: + deploy_path: ./deploy/tg/ecs + environment: ${{ vars.ENVIRONMENT }} + run: | + params=( apps/alb/${{ vars.ALB }} apps/ecr/${{ vars.ECR_REPOSITORY }} core rds/${{ vars.RDS_DB }} ) + for param in ${params[@]}; do + filename="$environment.${param//\//-}.auto.tfvars.json" + aws ssm get-parameters-by-path \ + --path "/$param/" \ + --recursive \ + --output json \ + --query 'Parameters[*]' \ + | jq '. |= map({ (.Name | split("/")[-1]): .Value }) | add' \ + > "$deploy_path/$filename" + done + - name: Expose github environment as shell variables env: SECRETS_CONTEXT: ${{ toJson(secrets) }} VARS_CONTEXT: ${{ toJson(vars) }} run: | - # https://docs.github.com/en/actions/using-workflows/workflow-commands-for-github-actions#setting-an-environment-variable - # https://docs.github.com/en/actions/using-workflows/workflow-commands-for-github-actions#multiline-strings EOF=$(dd if=/dev/urandom bs=15 count=1 status=none | base64) to_envs() { jq -r "to_entries[] | \"\(.key)<<$EOF\n\(.value)\n$EOF\n\""; } echo "$VARS_CONTEXT" | to_envs >> $GITHUB_ENV @@ -122,6 +130,8 @@ jobs: env: TF_INPUT: 0 TF_IN_AUTOMATION: true + # get the image digest from the build job with optional override from vars context + TF_VAR_image_tag: ${{ vars.IMAGE || needs.build_test_push.outputs.image_digest }} staging_deploy_apply: runs-on: ubuntu-latest @@ -132,26 +142,40 @@ jobs: uses: actions/checkout@v4 - name: Configure AWS Credentials - if: ${{ !env.ACT }} uses: aws-actions/configure-aws-credentials@v4 with: audience: sts.amazonaws.com aws-region: ${{ vars.AWS_REGION }} role-to-assume: ${{ secrets.AWS_ROLE_ARN }} + - name: Export shared infrastructure SSM parameter values to auto.tfvars.json files + env: + deploy_path: ./deploy/tg/ecs + environment: ${{ vars.ENVIRONMENT }} + run: | + params=( apps/alb/${{ vars.ALB }} apps/ecr/${{ vars.ECR_REPOSITORY }} core rds/${{ vars.RDS_DB }} ) + for param in ${params[@]}; do + filename="$environment.${param//\//-}.auto.tfvars.json" + aws ssm get-parameters-by-path \ + --path "/$param/" \ + --recursive \ + --output json \ + --query 'Parameters[*]' \ + | jq '. |= map({ (.Name | split("/")[-1]): .Value }) | add' \ + > "$deploy_path/$filename" + done + - name: Expose github environment as shell variables env: SECRETS_CONTEXT: ${{ toJson(secrets) }} VARS_CONTEXT: ${{ toJson(vars) }} run: | - # https://docs.github.com/en/actions/using-workflows/workflow-commands-for-github-actions#setting-an-environment-variable - # https://docs.github.com/en/actions/using-workflows/workflow-commands-for-github-actions#multiline-strings EOF=$(dd if=/dev/urandom bs=15 count=1 status=none | base64) to_envs() { jq -r "to_entries[] | \"\(.key)<<$EOF\n\(.value)\n$EOF\n\""; } echo "$VARS_CONTEXT" | to_envs >> $GITHUB_ENV echo "$SECRETS_CONTEXT" | to_envs >> $GITHUB_ENV - - name: Terragrunt Apply + - name: Terragrunt Plan id: terragrunt_plan uses: gruntwork-io/terragrunt-action@v2.0.0 with: @@ -162,3 +186,5 @@ jobs: env: TF_INPUT: 0 TF_IN_AUTOMATION: true + # get the image digest from the build job with optional override from vars context + TF_VAR_image_tag: ${{ vars.IMAGE || needs.build_test_push.outputs.image_digest }} \ No newline at end of file From 89105c6af4698e5aedc428696e6956fad8a6ca0f Mon Sep 17 00:00:00 2001 From: digorgonzola <29941279+digorgonzola@users.noreply.github.com> Date: Fri, 8 Dec 2023 23:35:14 +1100 Subject: [PATCH 37/55] tf/ecs: update vars to match ssm parameter key names --- deploy/tf/ecs/.terraform.lock.hcl | 25 -------------- deploy/tf/ecs/alb.tf | 4 +-- deploy/tf/ecs/ecs.tf | 42 ++++++++++++++++-------- deploy/tf/ecs/variables.tf | 54 +++++++++++++++++-------------- 4 files changed, 60 insertions(+), 65 deletions(-) delete mode 100644 deploy/tf/ecs/.terraform.lock.hcl diff --git a/deploy/tf/ecs/.terraform.lock.hcl b/deploy/tf/ecs/.terraform.lock.hcl deleted file mode 100644 index eabf303..0000000 --- a/deploy/tf/ecs/.terraform.lock.hcl +++ /dev/null @@ -1,25 +0,0 @@ -# This file is maintained automatically by "terraform init". -# Manual edits may be lost in future updates. - -provider "registry.terraform.io/hashicorp/aws" { - version = "5.29.0" - constraints = ">= 4.66.1" - hashes = [ - "h1:SyiKAX/D3ZE9My7P03DrRMf65pNnfSDQXPb0g11lCS0=", - "zh:0453c1c64e51cd7050ce46d9280a0195b9073592508077ebf1a1c45f7026f3f5", - "zh:3ee87d1a2870b61fdcc80f3f96b669dbcc8171aadb821bec0e1fa0e6fb9595b6", - "zh:423c0304eba345167cc37dcd300712f24f03fe4de8eecc15edb0d4f88b29ec79", - "zh:6816ce0ed702263297a8e02467bb712c509a9f6e4f132a152a10f1cc19191a81", - "zh:6feb8a0aedabd778216238e72273f5c2ee86d8841acc3fb3dc9d8014a2bbdc51", - "zh:709ccdc8b37f975d422e7955814671548887613931e234e06249da629b0f2f95", - "zh:76c55744020dbdafea25be634f8ac37c1e371f8c397f73bd89bc270d00ee0834", - "zh:7e48d6fc488b9dbe2fd4bebefa1b485d04da38b11a6799f8cba178173b7f8782", - "zh:951d7ef2adbfb96b1d3e9c4780b2ab4375caf9c6b522a2d023c02ff0698d8e2a", - "zh:9b12af85486a96aedd8d7984b0ff811a4b42e3d88dad1a3fb4c0b580d04fa425", - "zh:b0bf5974bc1a7d2ce3f3a9a31a8238ad15ad02211f1e84c54832541ec4bd5d10", - "zh:cc56d4ab9bcbee95f73dbe90f11d4ff7299b835dddf2b30cfda526a2cccd0f9f", - "zh:cfe3a4394f2f7044e03bb63f4fb9c691926607c6784417ac9c0724943da60d09", - "zh:d6f82e13f33f70de8df480287b5a961ced5606f041d1c589f706b112f68db890", - "zh:fb7be5bcff62d0ca9edd4a1bee4d2ed16e9428e3f9eff3ea4d898ecb234505a3", - ] -} diff --git a/deploy/tf/ecs/alb.tf b/deploy/tf/ecs/alb.tf index 6882f51..0dd9c69 100644 --- a/deploy/tf/ecs/alb.tf +++ b/deploy/tf/ecs/alb.tf @@ -13,7 +13,7 @@ resource "aws_lb_target_group" "app" { resource "aws_route53_record" "app" { for_each = toset(var.app_hostnames) - zone_id = var.dns_zone_id + zone_id = var.zone_id name = each.value type = "A" @@ -26,7 +26,7 @@ resource "aws_route53_record" "app" { resource "aws_lb_listener_rule" "app_fgate" { for_each = toset(var.app_hostnames) - listener_arn = var.alb_listener_arn + listener_arn = var.alb_https_listener_arn action { type = "forward" diff --git a/deploy/tf/ecs/ecs.tf b/deploy/tf/ecs/ecs.tf index 121d1e6..cfab5c6 100644 --- a/deploy/tf/ecs/ecs.tf +++ b/deploy/tf/ecs/ecs.tf @@ -1,3 +1,19 @@ +locals { + # set container definition variables with default fallback values from ssm if available + allowed_hosts = var.allowed_hosts + allowed_cidr_nets = coalesce(var.allowed_cidr_nets, var.subnets_private_cidr) + django_secret_key = var.django_secret_key + db_host = coalesce(var.db_host, var.rds_url) + db_name = var.db_name + db_user = var.db_user + db_secret_name = var.db_secret_name + db_secret_region = var.db_secret_region + s3_storage_bucket_name = var.s3_storage_bucket_name + s3_storage_bucket_region = var.s3_storage_bucket_region + + ecr_registry = split("/", var.ecr_repository_url)[0] +} + module "ecs" { source = "terraform-aws-modules/ecs/aws" version = "~> 5.7.0" @@ -50,7 +66,7 @@ module "ecs" { container_definitions = { api = { name = "api" - image = "${var.ecr_registry}/api:${var.image_tag}" + image = startswith(var.image, "sha256") ? "${var.ecr_repository_url}@${var.image}" : "${var.ecr_repository_url}:${var.image}" health_check = { command = ["CMD-SHELL", "uwsgi-is-ready --stats-socket /tmp/statsock > /dev/null 2>&1 || exit 1"] } @@ -58,16 +74,16 @@ module "ecs" { essential = true memory_reservation = 256 environment = [ - { name = "ALLOWED_HOSTS", value = var.allowed_hosts }, - { name = "ALLOWED_CIDR_NETS", value = var.allowed_cidr_nets }, - { name = "DJANGO_SECRET_KEY", value = var.django_secret_key }, - { name = "DB_HOST", value = var.db_host }, - { name = "DB_NAME", value = var.db_name }, - { name = "DB_USER", value = var.db_user }, - { name = "DB_SECRET_NAME", value = var.db_secret_name }, - { name = "DB_SECRET_REGION", value = var.db_secret_region }, - { name = "S3_STORAGE_BUCKET_NAME", value = var.s3_storage_bucket_name }, - { name = "S3_STORAGE_BUCKET_REGION", value = var.s3_storage_bucket_region } + { name = "ALLOWED_HOSTS", value = local.allowed_hosts }, + { name = "ALLOWED_CIDR_NETS", value = local.allowed_cidr_nets }, + { name = "DJANGO_SECRET_KEY", value = local.django_secret_key }, + { name = "DB_HOST", value = local.db_host }, + { name = "DB_NAME", value = local.db_name }, + { name = "DB_USER", value = local.db_user }, + { name = "DB_SECRET_NAME", value = local.db_secret_name }, + { name = "DB_SECRET_REGION", value = local.db_secret_region }, + { name = "S3_STORAGE_BUCKET_NAME", value = local.s3_storage_bucket_name }, + { name = "S3_STORAGE_BUCKET_REGION", value = local.s3_storage_bucket_region } ] port_mappings = [ { @@ -86,7 +102,7 @@ module "ecs" { } proxy = { name = "proxy" - image = "${var.ecr_registry}/nginx-proxy:latest" + image = "${local.ecr_registry}/nginx-proxy:latest" health_check = { command = ["CMD-SHELL", "curl -so /dev/null http://localhost/health || exit 1"] } @@ -128,7 +144,7 @@ module "ecs" { } } - subnet_ids = var.subnet_ids + subnet_ids = split(",", var.subnets_private) security_group_rules = { ingress_vpc = { diff --git a/deploy/tf/ecs/variables.tf b/deploy/tf/ecs/variables.tf index 0c124c5..f065aa4 100644 --- a/deploy/tf/ecs/variables.tf +++ b/deploy/tf/ecs/variables.tf @@ -1,16 +1,16 @@ variable "alb_dns_name" { description = "The DNS name of the application load-balancer" - type = string + type = string } -variable "alb_zone_id" { - description = "The DNS zone ID of the application load-balancer" - type = string +variable "alb_https_listener_arn" { + description = "The ARN of the application load-balancer listener" + type = string } -variable "alb_listener_arn" { - description = "The ARN of the application load-balancer listener" - type = string +variable "alb_zone_id" { + description = "The DNS zone ID of the application load-balancer" + type = string } variable "app_hostnames" { @@ -24,25 +24,19 @@ variable "container_port" { default = 80 } -variable "dns_zone_id" { - description = "The ID of the route53 zone" - type = string -} - -variable "ecr_registry" { - description = "The URL of the docker registry" +variable "db_host" { + description = "Override variable for database host" type = string } -variable "ecr_repository" { +variable "ecr_repository_url" { description = "The name of the repository to pull the image from" type = string } -variable "image_tag" { - description = "The tag of the docker image to pull from ECR" +variable "image" { + description = "The digest/tag of the docker image to pull from ECR" type = string - default = "latest" } variable "environment" { @@ -50,9 +44,19 @@ variable "environment" { type = string } -variable "subnet_ids" { +variable "rds_url" { + description = "The hostname of the database instance" + type = string +} + +variable "subnets_private" { description = "ID's of the subnets to deploy the container too" - type = list(string) + type = string +} + +variable "subnets_private_cidr" { + description = "CIDR's of the subnets to deploy the container too" + type = string } variable "vpc_cidr" { @@ -65,6 +69,11 @@ variable "vpc_id" { type = string } +variable "zone_id" { + description = "The ID of the route53 zone" + type = string +} + # Container environment variables variable "allowed_hosts" { description = "Hosts allowed to access the application container (i.e. 127.0.0.1)" @@ -81,11 +90,6 @@ variable "django_secret_key" { type = string } -variable "db_host" { - description = "The hostname of the database instance" - type = string -} - variable "db_name" { description = "The name of the database" type = string From 2778cdb4f4fe1c66dee2d65e82ea320506a7a1a7 Mon Sep 17 00:00:00 2001 From: digorgonzola <29941279+digorgonzola@users.noreply.github.com> Date: Fri, 8 Dec 2023 23:35:33 +1100 Subject: [PATCH 38/55] tg: remove ssm-parameter module --- deploy/tg/ssm-parameter/.terraform.lock.hcl | 24 --------------------- deploy/tg/ssm-parameter/terragrunt.hcl | 16 -------------- 2 files changed, 40 deletions(-) delete mode 100644 deploy/tg/ssm-parameter/.terraform.lock.hcl delete mode 100644 deploy/tg/ssm-parameter/terragrunt.hcl diff --git a/deploy/tg/ssm-parameter/.terraform.lock.hcl b/deploy/tg/ssm-parameter/.terraform.lock.hcl deleted file mode 100644 index 47d4158..0000000 --- a/deploy/tg/ssm-parameter/.terraform.lock.hcl +++ /dev/null @@ -1,24 +0,0 @@ -# This file is maintained automatically by "terraform init". -# Manual edits may be lost in future updates. - -provider "registry.terraform.io/hashicorp/aws" { - version = "5.29.0" - hashes = [ - "h1:+4qYlyPaktjZTFP9UbpEaz55jfwWapdibViCPoRFf+s=", - "zh:0453c1c64e51cd7050ce46d9280a0195b9073592508077ebf1a1c45f7026f3f5", - "zh:3ee87d1a2870b61fdcc80f3f96b669dbcc8171aadb821bec0e1fa0e6fb9595b6", - "zh:423c0304eba345167cc37dcd300712f24f03fe4de8eecc15edb0d4f88b29ec79", - "zh:6816ce0ed702263297a8e02467bb712c509a9f6e4f132a152a10f1cc19191a81", - "zh:6feb8a0aedabd778216238e72273f5c2ee86d8841acc3fb3dc9d8014a2bbdc51", - "zh:709ccdc8b37f975d422e7955814671548887613931e234e06249da629b0f2f95", - "zh:76c55744020dbdafea25be634f8ac37c1e371f8c397f73bd89bc270d00ee0834", - "zh:7e48d6fc488b9dbe2fd4bebefa1b485d04da38b11a6799f8cba178173b7f8782", - "zh:951d7ef2adbfb96b1d3e9c4780b2ab4375caf9c6b522a2d023c02ff0698d8e2a", - "zh:9b12af85486a96aedd8d7984b0ff811a4b42e3d88dad1a3fb4c0b580d04fa425", - "zh:b0bf5974bc1a7d2ce3f3a9a31a8238ad15ad02211f1e84c54832541ec4bd5d10", - "zh:cc56d4ab9bcbee95f73dbe90f11d4ff7299b835dddf2b30cfda526a2cccd0f9f", - "zh:cfe3a4394f2f7044e03bb63f4fb9c691926607c6784417ac9c0724943da60d09", - "zh:d6f82e13f33f70de8df480287b5a961ced5606f041d1c589f706b112f68db890", - "zh:fb7be5bcff62d0ca9edd4a1bee4d2ed16e9428e3f9eff3ea4d898ecb234505a3", - ] -} diff --git a/deploy/tg/ssm-parameter/terragrunt.hcl b/deploy/tg/ssm-parameter/terragrunt.hcl deleted file mode 100644 index 731f501..0000000 --- a/deploy/tg/ssm-parameter/terragrunt.hcl +++ /dev/null @@ -1,16 +0,0 @@ -include "global" { - path = find_in_parent_folders("terragrunt.hcl") - expose = true -} - -inputs = { - parameter_name = get_env("SSM_PARAMETER_NAME", "/apps/shared/devops/sydney") -} - -locals { - global = include.global.locals -} - -terraform { - source = "${get_repo_root()}//deploy/tf/ssm-parameter" -} From fe3c99f6764454c521924d40b8bcab0b4a7550eb Mon Sep 17 00:00:00 2001 From: digorgonzola <29941279+digorgonzola@users.noreply.github.com> Date: Fri, 8 Dec 2023 23:35:56 +1100 Subject: [PATCH 39/55] tg/ecs: remove obsolete dependencies and inputs --- deploy/tg/ecs/terragrunt.hcl | 61 ++++++------------------------------ 1 file changed, 9 insertions(+), 52 deletions(-) diff --git a/deploy/tg/ecs/terragrunt.hcl b/deploy/tg/ecs/terragrunt.hcl index 37742c6..a5fa112 100644 --- a/deploy/tg/ecs/terragrunt.hcl +++ b/deploy/tg/ecs/terragrunt.hcl @@ -1,32 +1,3 @@ -dependency "ssm" { - config_path = "../ssm-parameter" - mock_outputs = { - parameter_values = { - "alb_arn" = "arn:aws:elasticloadbalancing:ap-southeast-2:450356697252:loadbalancer/app/shared-alb-devops-sydney/45b0c41ea845014b" - "alb_dns_name" = "shared-alb-devops-sydney-387767645.ap-southeast-2.elb.amazonaws.com" - "alb_https_listener" = "arn:aws:elasticloadbalancing:ap-southeast-2:450356697252:listener/app/shared-alb-devops-sydney/45b0c41ea845014b/ecf16c7cec52b0c9" - "alb_zone_id" = "Z1GM3OXH4ZPM65" - "dns_domain_name" = "gamma.aodn.org.au" - "dns_zone_id" = "Z03033261P68C2JNNSWPQ" - "rds_url" = "stefan-db-rds-primary-evaluation.gamma.aodn.org.au" - "vpc_cidr" = "10.32.0.0/16" - "vpc_id" = "vpc-006dcec5d49e40003" - "vpc_private_subnets" = [ - "subnet-06665685d5a875465", - "subnet-0df67c37ed85ed7f2", - "subnet-075ea5fbe179b2b9a", - ] - "vpc_subnet_cidrs" = { - "private" = [ - "10.32.48.0/20", - "10.32.64.0/20", - "10.32.80.0/20", - ] - } - } - } -} - dependency "s3" { config_path = "../s3" mock_outputs = { @@ -51,30 +22,16 @@ inputs = { # DNS hostnames to associate with the container app_hostnames = ["api-${local.global.environment}"] - # container image repository and tag - ecr_registry = get_env("ECR_REGISTRY", "450356697252.dkr.ecr.ap-southeast-2.amazonaws.com") - ecr_repository = get_env("ECR_REPOSITORY", "api") - image_tag = get_env("IMAGE_TAG", "latest") - - # Shared infrastructure details - alb_dns_name = dependency.ssm.outputs.parameter_values.alb_dns_name - alb_listener_arn = dependency.ssm.outputs.parameter_values.alb_https_listener - alb_zone_id = dependency.ssm.outputs.parameter_values.alb_zone_id - dns_zone_id = dependency.ssm.outputs.parameter_values.dns_zone_id - subnet_ids = dependency.ssm.outputs.parameter_values.vpc_private_subnets - vpc_cidr = dependency.ssm.outputs.parameter_values.vpc_cidr - vpc_id = dependency.ssm.outputs.parameter_values.vpc_id - # get docker environment variable values with default fallback values - allowed_hosts = get_env("ALLOWED_HOSTS", "*") - allowed_cidr_nets = get_env("ALLOWED_CIDR_NETS", join(",", dependency.ssm.outputs.parameter_values.vpc_subnet_cidrs.private)) - django_secret_key = get_env("DJANGO_SECRET_KEY", "changeme") - db_host = get_env("DB_HOST", dependency.ssm.outputs.parameter_values.rds_url) - db_name = get_env("DB_NAME", "api") - db_user = get_env("DB_USER", "api") - db_secret_name = get_env("DB_SECRET_NAME", "/rds/stefan-db/primary/evaluation/api") - db_secret_region = get_env("DB_SECRET_REGION", "ap-southeast-2") - s3_storage_bucket_name = get_env("S3_STORAGE_BUCKET_NAME", + allowed_hosts = get_env("ALLOWED_HOSTS", "*") + allowed_cidr_nets = get_env("ALLOWED_CIDR_NETS", "") + django_secret_key = get_env("DJANGO_SECRET_KEY", "changeme") + db_host = get_env("DB_HOST", "") + db_name = get_env("DB_NAME", "api") + db_user = get_env("DB_USER", "api") + db_secret_name = get_env("DB_SECRET_NAME", "/rds/stefan-db/primary/evaluation/api") + db_secret_region = get_env("DB_SECRET_REGION", "ap-southeast-2") + s3_storage_bucket_name = get_env("S3_STORAGE_BUCKET_NAME", dependency.s3.outputs.wrapper["sample-django-app-${local.global.environment}-${local.global.aws_account}"].s3_bucket_id) s3_storage_bucket_region = get_env("S3_STORAGE_BUCKET_REGION", dependency.s3.outputs.wrapper["sample-django-app-${local.global.environment}-${local.global.aws_account}"].s3_bucket_region) From ece812bbcb941942ead93c02fe62d1bdf2cca82e Mon Sep 17 00:00:00 2001 From: digorgonzola <29941279+digorgonzola@users.noreply.github.com> Date: Fri, 8 Dec 2023 23:42:03 +1100 Subject: [PATCH 40/55] test.yml: correct image var name --- .github/workflows/test.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 01b9d37..b4a4465 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -131,7 +131,7 @@ jobs: TF_INPUT: 0 TF_IN_AUTOMATION: true # get the image digest from the build job with optional override from vars context - TF_VAR_image_tag: ${{ vars.IMAGE || needs.build_test_push.outputs.image_digest }} + TF_VAR_image: ${{ vars.IMAGE || needs.build_test_push.outputs.image_digest }} staging_deploy_apply: runs-on: ubuntu-latest @@ -187,4 +187,4 @@ jobs: TF_INPUT: 0 TF_IN_AUTOMATION: true # get the image digest from the build job with optional override from vars context - TF_VAR_image_tag: ${{ vars.IMAGE || needs.build_test_push.outputs.image_digest }} \ No newline at end of file + TF_VAR_image: ${{ vars.IMAGE || needs.build_test_push.outputs.image_digest }} \ No newline at end of file From 77e708b8c4e5b6716e3a0194b84c7cbae6241a1d Mon Sep 17 00:00:00 2001 From: digorgonzola <29941279+digorgonzola@users.noreply.github.com> Date: Fri, 8 Dec 2023 23:52:35 +1100 Subject: [PATCH 41/55] test.yml: update step name and add new line --- .github/workflows/test.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index b4a4465..24a2ec1 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -175,7 +175,7 @@ jobs: echo "$VARS_CONTEXT" | to_envs >> $GITHUB_ENV echo "$SECRETS_CONTEXT" | to_envs >> $GITHUB_ENV - - name: Terragrunt Plan + - name: Terragrunt Apply id: terragrunt_plan uses: gruntwork-io/terragrunt-action@v2.0.0 with: @@ -187,4 +187,4 @@ jobs: TF_INPUT: 0 TF_IN_AUTOMATION: true # get the image digest from the build job with optional override from vars context - TF_VAR_image: ${{ vars.IMAGE || needs.build_test_push.outputs.image_digest }} \ No newline at end of file + TF_VAR_image: ${{ vars.IMAGE || needs.build_test_push.outputs.image_digest }} From 7fd9cf103435417425f7204e52c25f91d65e7e8e Mon Sep 17 00:00:00 2001 From: digorgonzola <29941279+digorgonzola@users.noreply.github.com> Date: Sat, 9 Dec 2023 00:17:12 +1100 Subject: [PATCH 42/55] terragrunt.hcl: add repo url to tags --- deploy/tg/terragrunt.hcl | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/deploy/tg/terragrunt.hcl b/deploy/tg/terragrunt.hcl index b2b13d4..fc61db7 100644 --- a/deploy/tg/terragrunt.hcl +++ b/deploy/tg/terragrunt.hcl @@ -1,8 +1,9 @@ locals { aws_account = get_env("AWS_ACCOUNT_ID") - aws_region = get_env("AWS_REGION", "ap-southeast-2") + aws_region = get_env("AWS_REGION") environment = get_env("ENVIRONMENT") project_name = "sample-django-app" + repo_url = run_cmd("--terragrunt-quiet", "sh", "-c", "git config --get remote.origin.url") state_bucket = "tfstate-${local.aws_account}-${local.aws_region}" state_key = "apps/${local.project_name}/${local.environment}/${basename(get_terragrunt_dir())}.tfstate" } @@ -17,9 +18,10 @@ provider "aws" { default_tags { tags = { "Environment" = "apps" - "ManagedBy" = "Terradeploy Apps - ${local.state_bucket}/${local.state_key}" + "ManagedBy" = "Apps - ${local.state_bucket}/${local.state_key}" "Owner" = "Platform Engineering" "Project" = "AODN Applications" + "Repository" = "${local.repo_url}" } } } From 4871c3a81ae07a750d29ad4cd3f9e3e2a385c11e Mon Sep 17 00:00:00 2001 From: digorgonzola <29941279+digorgonzola@users.noreply.github.com> Date: Sat, 9 Dec 2023 22:30:42 +1100 Subject: [PATCH 43/55] test.yml: just build and test docker image --- .github/workflows/test.yml | 157 +------------------------------------ 1 file changed, 4 insertions(+), 153 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 24a2ec1..495230a 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -1,12 +1,9 @@ -name: Build, Test and Deploy +name: Build and Test on: pull_request: branches: - - "master" - -env: - AWS_REGION: ap-southeast-2 + - master permissions: id-token: write # This is required for requesting the JWT @@ -15,9 +12,6 @@ permissions: jobs: build_test_push: runs-on: ubuntu-latest - environment: staging - outputs: - image_digest: ${{ steps.build_and_push.outputs.digest }} steps: - name: Checkout uses: actions/checkout@v4 @@ -28,15 +22,6 @@ jobs: - name: Set up Docker Buildx uses: docker/setup-buildx-action@v3 - - name: Set Image Tag - id: set_tag - run: | - BRANCH_NAME=${{ github.head_ref || github.ref_name }} - echo TAG=${{ env.TAG_PREFIX }}-${BRANCH_NAME//\//_} >> $GITHUB_ENV - echo "image_tag=${{ env.TAG_PREFIX}}-${BRANCH_NAME/\//_}" >> $GITHUB_OUTPUT - env: - TAG_PREFIX: dev - - name: Setup Docker Structure Test run: > curl -LO @@ -44,147 +29,13 @@ jobs: && chmod +x container-structure-test-linux-amd64 && sudo mv container-structure-test-linux-amd64 /usr/local/bin/container-structure-test - - name: Configure AWS Credentials - uses: aws-actions/configure-aws-credentials@v4 - with: - audience: sts.amazonaws.com - aws-region: ${{ env.AWS_REGION }} - role-to-assume: ${{ secrets.AWS_ROLE_ARN }} - - - name: Login to ECR - uses: docker/login-action@v3 - with: - registry: ${{ vars.ECR_REGISTRY }} - - name: Build Docker Image uses: docker/build-push-action@v5 with: context: . load: true - tags: ${{ vars.ECR_REPOSITORY }}:${{ env.TAG }} + tags: ${{ vars.ECR_REPOSITORY }}:${{ github.ref }} - name: Test Docker Image run: | - container-structure-test test --image ${{ vars.ECR_REPOSITORY }}:${{ env.TAG }} --config tests/config.yaml - - - name: Build and Push Docker Image - id: build_and_push - uses: docker/build-push-action@v5 - with: - context: . -# Only building for AMD64 for now -# platforms: linux/amd64,linux/arm64 - push: true - tags: ${{ vars.ECR_REGISTRY }}/${{ vars.ECR_REPOSITORY }}:${{ env.TAG }} - - staging_deploy_plan: - runs-on: ubuntu-latest - environment: staging - needs: build_test_push - steps: - - name: Checkout - uses: actions/checkout@v4 - - - name: Configure AWS Credentials - uses: aws-actions/configure-aws-credentials@v4 - with: - audience: sts.amazonaws.com - aws-region: ${{ vars.AWS_REGION }} - role-to-assume: ${{ secrets.AWS_ROLE_ARN }} - - - name: Export shared infrastructure SSM parameter values to auto.tfvars.json files - env: - deploy_path: ./deploy/tg/ecs - environment: ${{ vars.ENVIRONMENT }} - run: | - params=( apps/alb/${{ vars.ALB }} apps/ecr/${{ vars.ECR_REPOSITORY }} core rds/${{ vars.RDS_DB }} ) - for param in ${params[@]}; do - filename="$environment.${param//\//-}.auto.tfvars.json" - aws ssm get-parameters-by-path \ - --path "/$param/" \ - --recursive \ - --output json \ - --query 'Parameters[*]' \ - | jq '. |= map({ (.Name | split("/")[-1]): .Value }) | add' \ - > "$deploy_path/$filename" - done - - - name: Expose github environment as shell variables - env: - SECRETS_CONTEXT: ${{ toJson(secrets) }} - VARS_CONTEXT: ${{ toJson(vars) }} - run: | - EOF=$(dd if=/dev/urandom bs=15 count=1 status=none | base64) - to_envs() { jq -r "to_entries[] | \"\(.key)<<$EOF\n\(.value)\n$EOF\n\""; } - echo "$VARS_CONTEXT" | to_envs >> $GITHUB_ENV - echo "$SECRETS_CONTEXT" | to_envs >> $GITHUB_ENV - - - name: Terragrunt Plan - id: terragrunt_plan - uses: gruntwork-io/terragrunt-action@v2.0.0 - with: - tf_version: '1.5.7' - tg_version: '0.51.0' - tg_dir: './deploy/tg' - tg_command: 'run-all plan' - env: - TF_INPUT: 0 - TF_IN_AUTOMATION: true - # get the image digest from the build job with optional override from vars context - TF_VAR_image: ${{ vars.IMAGE || needs.build_test_push.outputs.image_digest }} - - staging_deploy_apply: - runs-on: ubuntu-latest - environment: staging - needs: [staging_deploy_plan, build_test_push] - steps: - - name: Checkout - uses: actions/checkout@v4 - - - name: Configure AWS Credentials - uses: aws-actions/configure-aws-credentials@v4 - with: - audience: sts.amazonaws.com - aws-region: ${{ vars.AWS_REGION }} - role-to-assume: ${{ secrets.AWS_ROLE_ARN }} - - - name: Export shared infrastructure SSM parameter values to auto.tfvars.json files - env: - deploy_path: ./deploy/tg/ecs - environment: ${{ vars.ENVIRONMENT }} - run: | - params=( apps/alb/${{ vars.ALB }} apps/ecr/${{ vars.ECR_REPOSITORY }} core rds/${{ vars.RDS_DB }} ) - for param in ${params[@]}; do - filename="$environment.${param//\//-}.auto.tfvars.json" - aws ssm get-parameters-by-path \ - --path "/$param/" \ - --recursive \ - --output json \ - --query 'Parameters[*]' \ - | jq '. |= map({ (.Name | split("/")[-1]): .Value }) | add' \ - > "$deploy_path/$filename" - done - - - name: Expose github environment as shell variables - env: - SECRETS_CONTEXT: ${{ toJson(secrets) }} - VARS_CONTEXT: ${{ toJson(vars) }} - run: | - EOF=$(dd if=/dev/urandom bs=15 count=1 status=none | base64) - to_envs() { jq -r "to_entries[] | \"\(.key)<<$EOF\n\(.value)\n$EOF\n\""; } - echo "$VARS_CONTEXT" | to_envs >> $GITHUB_ENV - echo "$SECRETS_CONTEXT" | to_envs >> $GITHUB_ENV - - - name: Terragrunt Apply - id: terragrunt_plan - uses: gruntwork-io/terragrunt-action@v2.0.0 - with: - tf_version: '1.5.7' - tg_version: '0.51.0' - tg_dir: './deploy/tg' - tg_command: 'run-all apply' - env: - TF_INPUT: 0 - TF_IN_AUTOMATION: true - # get the image digest from the build job with optional override from vars context - TF_VAR_image: ${{ vars.IMAGE || needs.build_test_push.outputs.image_digest }} + container-structure-test test --image ${{ vars.ECR_REPOSITORY }}:${{ github.ref }} --config tests/config.yaml From b0ed43256a1ec7b2dcbdd90bae16e0338cd8692d Mon Sep 17 00:00:00 2001 From: digorgonzola <29941279+digorgonzola@users.noreply.github.com> Date: Sat, 9 Dec 2023 22:46:54 +1100 Subject: [PATCH 44/55] deploy-staging.yml: build and push workflow for staging performs the following: - builds + tests docker image - runs tg plan against staging environment - runs tg apply against staging environment - creates a draft release with stored docker metadata --- .github/workflows/deploy-staging.yml | 203 +++++++++++++++++++++++++++ 1 file changed, 203 insertions(+) create mode 100644 .github/workflows/deploy-staging.yml diff --git a/.github/workflows/deploy-staging.yml b/.github/workflows/deploy-staging.yml new file mode 100644 index 0000000..0fb2dc2 --- /dev/null +++ b/.github/workflows/deploy-staging.yml @@ -0,0 +1,203 @@ +name: Deploy to Staging + +on: + push: + branches: + - master + +permissions: + id-token: write # This is required for requesting the JWT + contents: read # This is required for actions/checkout + +jobs: + build_test_push: + runs-on: ubuntu-latest + environment: staging + outputs: + image_digest: ${{ steps.build_and_push.outputs.digest }} + image_metadata: ${{ steps.build_and_push.outputs.metadata }} + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Set up QEMU + uses: docker/setup-qemu-action@v3 + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + + - name: Setup Docker Structure Test + run: > + curl -LO + https://storage.googleapis.com/container-structure-test/latest/container-structure-test-linux-amd64 + && chmod +x container-structure-test-linux-amd64 && sudo mv container-structure-test-linux-amd64 + /usr/local/bin/container-structure-test + + - name: Configure AWS Credentials + uses: aws-actions/configure-aws-credentials@v4 + with: + audience: sts.amazonaws.com + aws-region: ${{ env.AWS_REGION }} + role-to-assume: ${{ secrets.AWS_ROLE_ARN }} + + - name: Login to ECR + uses: docker/login-action@v3 + with: + registry: ${{ vars.ECR_REGISTRY }} + + - name: Build Docker Image + uses: docker/build-push-action@v5 + with: + context: . + load: true + tags: ${{ vars.ECR_REPOSITORY }}:latest + + - name: Test Docker Image + run: | + container-structure-test test --image ${{ vars.ECR_REPOSITORY }}:latest --config tests/config.yaml + + - name: Build and Push Docker Image + id: build_and_push + uses: docker/build-push-action@v5 + with: + context: . +# Only building for AMD64 for now +# platforms: linux/amd64,linux/arm64 + push: true + tags: ${{ vars.ECR_REGISTRY }}/${{ vars.ECR_REPOSITORY }}:latest + + staging_deploy_plan: + runs-on: ubuntu-latest + environment: staging + needs: build_test_push + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Configure AWS Credentials + uses: aws-actions/configure-aws-credentials@v4 + with: + audience: sts.amazonaws.com + aws-region: ${{ vars.AWS_REGION }} + role-to-assume: ${{ secrets.AWS_ROLE_ARN }} + + - name: Export shared infrastructure SSM parameter values to auto.tfvars.json files + env: + deploy_path: ./deploy/tg/ecs + environment: ${{ vars.ENVIRONMENT }} + run: | + params=( apps/alb/${{ vars.ALB }} apps/ecr/${{ vars.ECR_REPOSITORY }} core rds/${{ vars.RDS_DB }} ) + for param in ${params[@]}; do + filename="$environment.${param//\//-}.auto.tfvars.json" + aws ssm get-parameters-by-path \ + --path "/$param/" \ + --recursive \ + --output json \ + --query 'Parameters[*]' \ + | jq '. |= map({ (.Name | split("/")[-1]): .Value }) | add' \ + > "$deploy_path/$filename" + done + + - name: Expose github environment as shell variables + env: + SECRETS_CONTEXT: ${{ toJson(secrets) }} + VARS_CONTEXT: ${{ toJson(vars) }} + run: | + EOF=$(dd if=/dev/urandom bs=15 count=1 status=none | base64) + to_envs() { jq -r "to_entries[] | \"\(.key)<<$EOF\n\(.value)\n$EOF\n\""; } + echo "$VARS_CONTEXT" | to_envs >> $GITHUB_ENV + echo "$SECRETS_CONTEXT" | to_envs >> $GITHUB_ENV + + - name: Terragrunt Plan + id: terragrunt_plan + uses: gruntwork-io/terragrunt-action@v2.0.0 + with: + tf_version: '1.5.7' + tg_version: '0.51.0' + tg_dir: './deploy/tg' + tg_command: 'run-all plan' + env: + TF_INPUT: 0 + TF_IN_AUTOMATION: true + # get the image digest from the build job with optional override from vars context + TF_VAR_image: ${{ vars.IMAGE || needs.build_test_push.outputs.image_digest }} + + staging_deploy_apply: + runs-on: ubuntu-latest + environment: staging + needs: [staging_deploy_plan, build_test_push] + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Configure AWS Credentials + uses: aws-actions/configure-aws-credentials@v4 + with: + audience: sts.amazonaws.com + aws-region: ${{ vars.AWS_REGION }} + role-to-assume: ${{ secrets.AWS_ROLE_ARN }} + + - name: Export shared infrastructure SSM parameter values to auto.tfvars.json files + env: + deploy_path: ./deploy/tg/ecs + environment: ${{ vars.ENVIRONMENT }} + run: | + params=( apps/alb/${{ vars.ALB }} apps/ecr/${{ vars.ECR_REPOSITORY }} core rds/${{ vars.RDS_DB }} ) + for param in ${params[@]}; do + filename="$environment.${param//\//-}.auto.tfvars.json" + aws ssm get-parameters-by-path \ + --path "/$param/" \ + --recursive \ + --output json \ + --query 'Parameters[*]' \ + | jq '. |= map({ (.Name | split("/")[-1]): .Value }) | add' \ + > "$deploy_path/$filename" + done + + - name: Expose github environment as shell variables + env: + SECRETS_CONTEXT: ${{ toJson(secrets) }} + VARS_CONTEXT: ${{ toJson(vars) }} + run: | + EOF=$(dd if=/dev/urandom bs=15 count=1 status=none | base64) + to_envs() { jq -r "to_entries[] | \"\(.key)<<$EOF\n\(.value)\n$EOF\n\""; } + echo "$VARS_CONTEXT" | to_envs >> $GITHUB_ENV + echo "$SECRETS_CONTEXT" | to_envs >> $GITHUB_ENV + + - name: Terragrunt Apply + id: terragrunt_plan + uses: gruntwork-io/terragrunt-action@v2.0.0 + with: + tf_version: '1.5.7' + tg_version: '0.51.0' + tg_dir: './deploy/tg' + tg_command: 'run-all apply' + env: + TF_INPUT: 0 + TF_IN_AUTOMATION: true + # get the image digest from the build job with optional override from vars context + TF_VAR_image: ${{ vars.IMAGE || needs.build_test_push.outputs.image_digest }} + + create_release: + name: Create Release + runs-on: ubuntu-latest + needs: [build_test_push] + steps: + - name: Write image metadata to file + id: metadata_to_file + run: ${{ needs.build_test_push.outputs.image_metadata }} > metadata.json + - name: Create Draft Release + id: create_draft_release + uses: softprops/action-gh-release@v1 + with: + name: Draft Release ${{ github.ref }} + body: | + ## Info + Commit ${{ github.sha }} was deployed to `staging`. [See code diff](${{ github.event.compare }}). + + It was initialized by [${{ github.event.sender.login }}](${{ github.event.sender.html_url }}). + + ## How to Promote? + In order to promote this to prod, edit the draft and press **"Publish release"**. + draft: true + files: metadata.json From 85d723eaef366e2b254852e463fa0ae5e250f699 Mon Sep 17 00:00:00 2001 From: digorgonzola <29941279+digorgonzola@users.noreply.github.com> Date: Sat, 9 Dec 2023 22:47:51 +1100 Subject: [PATCH 45/55] deploy-production.yml: plan and deploy to production performs the following: - gets the release metadata artifact - runs tg plan against production environment - runs tg apply against production environment --- .github/workflows/deploy-production.yml | 146 ++++++++++++++++++++++++ 1 file changed, 146 insertions(+) create mode 100644 .github/workflows/deploy-production.yml diff --git a/.github/workflows/deploy-production.yml b/.github/workflows/deploy-production.yml new file mode 100644 index 0000000..21d10a3 --- /dev/null +++ b/.github/workflows/deploy-production.yml @@ -0,0 +1,146 @@ +name: release + +on: + release: + types: + - published + +permissions: + id-token: write # This is required for requesting the JWT + contents: read # This is required for actions/checkout + +jobs: + production_deploy_plan: + runs-on: ubuntu-latest + environment: production + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Configure AWS Credentials + uses: aws-actions/configure-aws-credentials@v4 + with: + audience: sts.amazonaws.com + aws-region: ${{ vars.AWS_REGION }} + role-to-assume: ${{ secrets.AWS_ROLE_ARN }} + + - name: Get Image Metadata from Release + uses: dsaltares/fetch-gh-release-asset@v1.1.1 + with: + version: ${{ github.event.release.id }} + file: metadata.json + + - name: Set Image Digest from Metadata + id: set_image_digest + run: | + image_digest=$(cat /tmp/metadata.json | jq -r '."containerimage.digest"') + echo "image_digest=$image_digest" >> $GITHUB_OUTPUT + + - name: Export shared infrastructure SSM parameter values to auto.tfvars.json files + env: + deploy_path: ./deploy/tg/ecs + environment: ${{ vars.ENVIRONMENT }} + run: | + params=( apps/alb/${{ vars.ALB }} apps/ecr/${{ vars.ECR_REPOSITORY }} core rds/${{ vars.RDS_DB }} ) + for param in ${params[@]}; do + filename="$environment.${param//\//-}.auto.tfvars.json" + aws ssm get-parameters-by-path \ + --path "/$param/" \ + --recursive \ + --output json \ + --query 'Parameters[*]' \ + | jq '. |= map({ (.Name | split("/")[-1]): .Value }) | add' \ + > "$deploy_path/$filename" + done + + - name: Expose github environment as shell variables + env: + SECRETS_CONTEXT: ${{ toJson(secrets) }} + VARS_CONTEXT: ${{ toJson(vars) }} + run: | + EOF=$(dd if=/dev/urandom bs=15 count=1 status=none | base64) + to_envs() { jq -r "to_entries[] | \"\(.key)<<$EOF\n\(.value)\n$EOF\n\""; } + echo "$VARS_CONTEXT" | to_envs >> $GITHUB_ENV + echo "$SECRETS_CONTEXT" | to_envs >> $GITHUB_ENV + + - name: Terragrunt Plan + id: terragrunt_plan + uses: gruntwork-io/terragrunt-action@v2.0.0 + with: + tf_version: '1.5.7' + tg_version: '0.51.0' + tg_dir: './deploy/tg' + tg_command: 'run-all plan' + env: + TF_INPUT: 0 + TF_IN_AUTOMATION: true + # get the image digest from the build job with optional override from vars context + TF_VAR_image: ${{ vars.IMAGE || steps.set_image_digest.outputs.image_digest }} + + production_deploy_apply: + runs-on: ubuntu-latest + environment: staging + needs: [production_deploy_plan] + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Configure AWS Credentials + uses: aws-actions/configure-aws-credentials@v4 + with: + audience: sts.amazonaws.com + aws-region: ${{ vars.AWS_REGION }} + role-to-assume: ${{ secrets.AWS_ROLE_ARN }} + + - name: Get Image Metadata from Release + uses: dsaltares/fetch-gh-release-asset@v1.1.1 + with: + version: ${{ github.event.release.id }} + file: metadata.json + + - name: Set Image Digest from Metadata + id: set_image_digest + run: | + image_digest=$(cat /tmp/metadata.json | jq -r '."containerimage.digest"') + echo "image_digest=$image_digest" >> $GITHUB_OUTPUT + + - name: Export shared infrastructure SSM parameter values to auto.tfvars.json files + env: + deploy_path: ./deploy/tg/ecs + environment: ${{ vars.ENVIRONMENT }} + run: | + params=( apps/alb/${{ vars.ALB }} apps/ecr/${{ vars.ECR_REPOSITORY }} core rds/${{ vars.RDS_DB }} ) + for param in ${params[@]}; do + filename="$environment.${param//\//-}.auto.tfvars.json" + aws ssm get-parameters-by-path \ + --path "/$param/" \ + --recursive \ + --output json \ + --query 'Parameters[*]' \ + | jq '. |= map({ (.Name | split("/")[-1]): .Value }) | add' \ + > "$deploy_path/$filename" + done + + - name: Expose github environment as shell variables + env: + SECRETS_CONTEXT: ${{ toJson(secrets) }} + VARS_CONTEXT: ${{ toJson(vars) }} + run: | + EOF=$(dd if=/dev/urandom bs=15 count=1 status=none | base64) + to_envs() { jq -r "to_entries[] | \"\(.key)<<$EOF\n\(.value)\n$EOF\n\""; } + echo "$VARS_CONTEXT" | to_envs >> $GITHUB_ENV + echo "$SECRETS_CONTEXT" | to_envs >> $GITHUB_ENV + + - name: Terragrunt Apply + id: terragrunt_plan + uses: gruntwork-io/terragrunt-action@v2.0.0 + with: + tf_version: '1.5.7' + tg_version: '0.51.0' + tg_dir: './deploy/tg' + tg_command: 'run-all apply' + env: + TF_INPUT: 0 + TF_IN_AUTOMATION: true + # get the image digest from the build job with optional override from vars context + TF_VAR_image: ${{ vars.IMAGE || steps.set_image_digest.outputs.image_digest }} From d6dafd5127ec5716fcf030c127451fbe3c72fb7f Mon Sep 17 00:00:00 2001 From: digorgonzola <29941279+digorgonzola@users.noreply.github.com> Date: Sat, 9 Dec 2023 22:48:02 +1100 Subject: [PATCH 46/55] delete release.yml --- .github/workflows/release.yml | 115 ---------------------------------- 1 file changed, 115 deletions(-) delete mode 100644 .github/workflows/release.yml diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml deleted file mode 100644 index 63e6355..0000000 --- a/.github/workflows/release.yml +++ /dev/null @@ -1,115 +0,0 @@ -name: release - -on: - release: - types: - - published - -env: - AWS_REGION: ap-southeast-2 - FAMILY: api-service - TAG: ${{ github.event.release.tag_name }} - -permissions: - id-token: write # This is required for requesting the JWT - contents: read # This is required for actions/checkout - -jobs: - build_test_push: - runs-on: ubuntu-latest - steps: - - name: Checkout - uses: actions/checkout@v4 - - - name: Set up QEMU - uses: docker/setup-qemu-action@v3 - - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v3 - - - name: Set up docker structure test - run: > - curl -LO - https://storage.googleapis.com/container-structure-test/latest/container-structure-test-linux-amd64 - && chmod +x container-structure-test-linux-amd64 && sudo mv container-structure-test-linux-amd64 - /usr/local/bin/container-structure-test - - - name: Configure AWS Credentials - if: ${{ !env.ACT }} - uses: aws-actions/configure-aws-credentials@v4 - with: - audience: sts.amazonaws.com - aws-region: ${{ env.AWS_REGION }} - role-to-assume: ${{ secrets.AWS_ROLE_ARN }} - - - name: Login to ECR - if: ${{ !env.ACT }} - uses: docker/login-action@v3 - with: - registry: ${{ vars.ECR_REGISTRY }} - - - name: Build - uses: docker/build-push-action@v5 - with: - context: . - load: true - tags: ${{ env.TAG }} - - - name: Test - run: | - container-structure-test test --image ${{ env.TAG }} --config tests/config.yaml - - - name: Build and Push - if: ${{ !env.ACT }} - uses: docker/build-push-action@v5 - with: - context: . - platforms: linux/amd64,linux/arm64 - push: true - tags: | - ${{ vars.ECR_REGISTRY }}/${{ vars.ECR_REPOSITORY }}:${{ env.TAG }} - ${{ vars.ECR_REGISTRY }}/${{ vars.ECR_REPOSITORY }}:latest - - render_and_deploy: - runs-on: ubuntu-latest - needs: build_test_push - steps: - - name: Checkout - uses: actions/checkout@v4 - - - name: Configure AWS Credentials - if: ${{ !env.ACT }} - uses: aws-actions/configure-aws-credentials@v4 - with: - audience: sts.amazonaws.com - aws-region: ${{ env.AWS_REGION }} - role-to-assume: ${{ secrets.AWS_ROLE_ARN }} - - - name: Get Current Task Definition - id: get-current-task-definition - run: > - aws ecs describe-task-definition --task-definition ${{ env.FAMILY }} - --query taskDefinition > task-definition.json - - - name: Update API image tag - id: update-api-image-tag - uses: aws-actions/amazon-ecs-render-task-definition@v1 - with: - task-definition: task-definition.json - container-name: api - image: ${{ vars.ECR_REGISTRY }}/${{ vars.ECR_REPOSITORY }}:${{ env.TAG }} - - - name: Display Rendered Template - if: ${{ env.ACT }} - id: display-rendered-template - run: cat ${{ steps.update-api-image-tag.outputs.task-definition }} | jq -r - - - name: Deploy to Amazon ECS service - if: ${{ !env.ACT }} - uses: aws-actions/amazon-ecs-deploy-task-definition@v1 - with: - task-definition: ${{ steps.update-api-image-tag.outputs.task-definition }} - service: api-service - cluster: api-cluster - force-new-deployment: true - wait-for-service-stability: true From 4ad255968bb58f6e4a7e8899d0f2eedab306a643 Mon Sep 17 00:00:00 2001 From: digorgonzola <29941279+digorgonzola@users.noreply.github.com> Date: Sat, 9 Dec 2023 22:52:13 +1100 Subject: [PATCH 47/55] test.yml: fix docker build tag --- .github/workflows/test.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 495230a..9f44abc 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -10,7 +10,7 @@ permissions: contents: read # This is required for actions/checkout jobs: - build_test_push: + build_and_test: runs-on: ubuntu-latest steps: - name: Checkout @@ -34,8 +34,8 @@ jobs: with: context: . load: true - tags: ${{ vars.ECR_REPOSITORY }}:${{ github.ref }} + tags: ${{ vars.ECR_REPOSITORY }} - name: Test Docker Image run: | - container-structure-test test --image ${{ vars.ECR_REPOSITORY }}:${{ github.ref }} --config tests/config.yaml + container-structure-test test --image ${{ vars.ECR_REPOSITORY }} --config tests/config.yaml From b5343de7118c720a497659d8a29c71c8f21314b9 Mon Sep 17 00:00:00 2001 From: digorgonzola <29941279+digorgonzola@users.noreply.github.com> Date: Sat, 9 Dec 2023 23:05:57 +1100 Subject: [PATCH 48/55] deploy-staging.yml: fix aws_region var --- .github/workflows/deploy-staging.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/deploy-staging.yml b/.github/workflows/deploy-staging.yml index 0fb2dc2..4cb387d 100644 --- a/.github/workflows/deploy-staging.yml +++ b/.github/workflows/deploy-staging.yml @@ -37,7 +37,7 @@ jobs: uses: aws-actions/configure-aws-credentials@v4 with: audience: sts.amazonaws.com - aws-region: ${{ env.AWS_REGION }} + aws-region: ${{ vars.AWS_REGION }} role-to-assume: ${{ secrets.AWS_ROLE_ARN }} - name: Login to ECR From f2d238c82e84bfd6066467c0baa429151637d2e5 Mon Sep 17 00:00:00 2001 From: digorgonzola <29941279+digorgonzola@users.noreply.github.com> Date: Sat, 9 Dec 2023 23:06:19 +1100 Subject: [PATCH 49/55] pre-commit.yml: run pre-commit checks before allowing merge --- .github/workflows/pre-commit.yml | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) create mode 100644 .github/workflows/pre-commit.yml diff --git a/.github/workflows/pre-commit.yml b/.github/workflows/pre-commit.yml new file mode 100644 index 0000000..8f4ad96 --- /dev/null +++ b/.github/workflows/pre-commit.yml @@ -0,0 +1,20 @@ +name: Run Pre-commit Checks + +on: + pull_request: + branches: + - master + +permissions: + id-token: write # This is required for requesting the JWT + contents: read # This is required for actions/checkout + +jobs: + pre_commit: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: + python-version: '3.10' + - uses: pre-commit/action@v3.0.0 From 099305935e4b069b22458327ec9be4b11dfa6da4 Mon Sep 17 00:00:00 2001 From: digorgonzola <29941279+digorgonzola@users.noreply.github.com> Date: Sat, 9 Dec 2023 23:14:00 +1100 Subject: [PATCH 50/55] pre-commit.yml: install terragrunt --- .github/workflows/pre-commit.yml | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/.github/workflows/pre-commit.yml b/.github/workflows/pre-commit.yml index 8f4ad96..f6d6ae6 100644 --- a/.github/workflows/pre-commit.yml +++ b/.github/workflows/pre-commit.yml @@ -17,4 +17,13 @@ jobs: - uses: actions/setup-python@v5 with: python-version: '3.10' + - name: Setup Terragrunt + id: setup_terragrunt + run: | + wget https://github.com/gruntwork-io/terragrunt/releases/download/v${tg_version}/terragrunt_linux_amd64 \ + && mv terragrunt_linux_amd64 terragrunt \ + && chmod +x terragrunt \ + && mv terragrunt /usr/local/bin/terragrunt + env: + tg_version: '0.51.0' - uses: pre-commit/action@v3.0.0 From fe6ea70d9d84b0e68af728f4021b5f5d66cf030a Mon Sep 17 00:00:00 2001 From: digorgonzola <29941279+digorgonzola@users.noreply.github.com> Date: Sat, 9 Dec 2023 23:20:42 +1100 Subject: [PATCH 51/55] requirements.txt: include pre-commit framework --- requirements.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/requirements.txt b/requirements.txt index 8629038..0cca336 100644 --- a/requirements.txt +++ b/requirements.txt @@ -10,3 +10,4 @@ Pillow>=10.1.0,<10.2.0 uwsgi>=2.0.23,<2.1.0 flake8>=6.1.0,<6.2.0 uwsgi-readiness-check>=0.2.0,<0.3.0 +pre-commit>=3.5.0,<3.6.0 From d1bc11a5aed0cd2a60f2182bdcc617f84dac5cba Mon Sep 17 00:00:00 2001 From: digorgonzola <29941279+digorgonzola@users.noreply.github.com> Date: Sat, 9 Dec 2023 23:32:14 +1100 Subject: [PATCH 52/55] .pre-commit-config.yaml: update tf_validate args --- .pre-commit-config.yaml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index efd206f..9780638 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -19,4 +19,6 @@ repos: hooks: - id: terraform_fmt - id: terraform_validate + args: + - --tf-init-args=-backend=false - id: terragrunt_fmt From c5f7bbbd6b80ea8e187458114ec557f272159ff1 Mon Sep 17 00:00:00 2001 From: digorgonzola <29941279+digorgonzola@users.noreply.github.com> Date: Sat, 9 Dec 2023 23:32:45 +1100 Subject: [PATCH 53/55] tf/ecs versions.tf: add required providers/tf version --- deploy/tf/ecs/versions.tf | 9 +++++++++ 1 file changed, 9 insertions(+) create mode 100644 deploy/tf/ecs/versions.tf diff --git a/deploy/tf/ecs/versions.tf b/deploy/tf/ecs/versions.tf new file mode 100644 index 0000000..91041e5 --- /dev/null +++ b/deploy/tf/ecs/versions.tf @@ -0,0 +1,9 @@ +terraform { + required_version = "~> 1.5.0" + required_providers { + aws = { + source = "hashicorp/aws" + version = "~> 5.12" + } + } +} From 0bb0d37f914539f299607bfa1f56d93fae8f0745 Mon Sep 17 00:00:00 2001 From: digorgonzola <29941279+digorgonzola@users.noreply.github.com> Date: Sat, 9 Dec 2023 23:33:05 +1100 Subject: [PATCH 54/55] tg/ecs: terragrunt.hcl update formatting --- deploy/tg/ecs/terragrunt.hcl | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/deploy/tg/ecs/terragrunt.hcl b/deploy/tg/ecs/terragrunt.hcl index a5fa112..d4c09d0 100644 --- a/deploy/tg/ecs/terragrunt.hcl +++ b/deploy/tg/ecs/terragrunt.hcl @@ -23,18 +23,18 @@ inputs = { app_hostnames = ["api-${local.global.environment}"] # get docker environment variable values with default fallback values - allowed_hosts = get_env("ALLOWED_HOSTS", "*") - allowed_cidr_nets = get_env("ALLOWED_CIDR_NETS", "") - django_secret_key = get_env("DJANGO_SECRET_KEY", "changeme") - db_host = get_env("DB_HOST", "") - db_name = get_env("DB_NAME", "api") - db_user = get_env("DB_USER", "api") - db_secret_name = get_env("DB_SECRET_NAME", "/rds/stefan-db/primary/evaluation/api") - db_secret_region = get_env("DB_SECRET_REGION", "ap-southeast-2") + allowed_hosts = get_env("ALLOWED_HOSTS", "*") + allowed_cidr_nets = get_env("ALLOWED_CIDR_NETS", "") + django_secret_key = get_env("DJANGO_SECRET_KEY", "changeme") + db_host = get_env("DB_HOST", "") + db_name = get_env("DB_NAME", "api") + db_user = get_env("DB_USER", "api") + db_secret_name = get_env("DB_SECRET_NAME", "/rds/stefan-db/primary/evaluation/api") + db_secret_region = get_env("DB_SECRET_REGION", "ap-southeast-2") s3_storage_bucket_name = get_env("S3_STORAGE_BUCKET_NAME", - dependency.s3.outputs.wrapper["sample-django-app-${local.global.environment}-${local.global.aws_account}"].s3_bucket_id) + dependency.s3.outputs.wrapper["sample-django-app-${local.global.environment}-${local.global.aws_account}"].s3_bucket_id) s3_storage_bucket_region = get_env("S3_STORAGE_BUCKET_REGION", - dependency.s3.outputs.wrapper["sample-django-app-${local.global.environment}-${local.global.aws_account}"].s3_bucket_region) + dependency.s3.outputs.wrapper["sample-django-app-${local.global.environment}-${local.global.aws_account}"].s3_bucket_region) } locals { From 72e33c2465b226af7fc0ed48c7602680c90dbf66 Mon Sep 17 00:00:00 2001 From: digorgonzola <29941279+digorgonzola@users.noreply.github.com> Date: Sat, 9 Dec 2023 23:36:46 +1100 Subject: [PATCH 55/55] pre-commit.yml: set tf version --- .github/workflows/pre-commit.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/workflows/pre-commit.yml b/.github/workflows/pre-commit.yml index f6d6ae6..fd69c98 100644 --- a/.github/workflows/pre-commit.yml +++ b/.github/workflows/pre-commit.yml @@ -17,6 +17,9 @@ jobs: - uses: actions/setup-python@v5 with: python-version: '3.10' + - uses: hashicorp/setup-terraform@v3 + with: + terraform_version: "1.5.7" - name: Setup Terragrunt id: setup_terragrunt run: |