major refactor to avoid the github variables mess

.github/workflows/deploy-development.yml

@@ -15,7 +15,7 @@ jobs:
     runs-on: ubuntu-latest
     environment: tf-development
     outputs:
-      image_tag: ${{ steps.set_image_tag.outputs.image_tag }}
+      image_digest: ${{ steps.build_and_push.outputs.digest }}
     steps:
       - name: Checkout
         uses: actions/checkout@v4
@@ -105,29 +105,13 @@ jobs:
           aws-region: ${{ vars.AWS_REGION }}
           role-to-assume: ${{ secrets.AWS_ROLE_ARN }}
 
-      - name: Expose github environment as shell variables
+      - name: Expose github variables to shell as environment variables
         env:
-          SECRETS_CONTEXT: ${{ toJson(secrets) }}
           VARS_CONTEXT: ${{ toJson(vars) }}
         run: |
           EOF=$(dd if=/dev/urandom bs=15 count=1 status=none | base64)
           to_envs() { jq -r "to_entries[] | \"\(.key)<<$EOF\n\(.value)\n$EOF\n\""; }
           echo "$VARS_CONTEXT" | to_envs >> $GITHUB_ENV
-          echo "$SECRETS_CONTEXT" | to_envs >> $GITHUB_ENV
-
-      - name: Export TF_VAR variables to environment with correct case
-        env:
-          vars_json: ${{ toJSON(vars) }}
-        run: |
-          tf_vars=$(echo $vars_json | jq -r '
-            . | with_entries( .key |= ascii_downcase | select(.key | startswith("tf_var")))
-              | to_entries
-              | map("TF_VAR_\(.key| split("tf_var_")[-1])=\(.value)") |.[]')
-
-          for var in "${tf_vars[@]}"; do
-            echo "$var"
-            echo "$var" >> $GITHUB_ENV
-          done
 
       - name: Terragrunt Plan
         uses: gruntwork-io/terragrunt-action@v2
@@ -140,7 +124,7 @@ jobs:
           TF_INPUT: 0
           TF_IN_AUTOMATION: true
           # get the image digest from the build job with optional override from vars context
-          TF_VAR_image: ${{ vars.IMAGE || needs.build_test_push.outputs.image_tag }}
+          TF_VAR_image: ${{ needs.build_test_push.outputs.image_digest }}
 
       - name: Terragrunt Apply
         uses: gruntwork-io/terragrunt-action@v2
@@ -153,4 +137,4 @@ jobs:
           TF_INPUT: 0
           TF_IN_AUTOMATION: true
           # get the image digest from the build job with optional override from vars context
-          TF_VAR_image: ${{ vars.IMAGE || needs.build_test_push.outputs.image_tag }}
+          TF_VAR_image: ${{ needs.build_test_push.outputs.image_digest }}

deploy/container/development.env

@@ -0,0 +1,8 @@
+ES_HOST=ec2-3-25-64-248.ap-southeast-2.compute.amazonaws.com
+INDEXER_HOST=ec2-3-25-163-152.ap-southeast-2.compute.amazonaws.com
+INDEXER_PORT=8081
+GEONETWORK_DB_TYPE=postgres
+GEONETWORK_DB_HOST=stefan-db-rds-primary-evaluation.gamma.aodn.org.au
+GEONETWORK_DB_PORT=5432
+GEONETWORK_DB_NAME=geonetwork
+GEONETWORK_DB_USERNAME=geonetwork

deploy/container/production.env

@@ -0,0 +1,8 @@
+ES_HOST=ec2-3-25-64-248.ap-southeast-2.compute.amazonaws.com
+INDEXER_HOST=ec2-3-25-163-152.ap-southeast-2.compute.amazonaws.com
+INDEXER_PORT=8081
+GEONETWORK_DB_TYPE=postgres
+GEONETWORK_DB_HOST=stefan-db-rds-primary-evaluation.gamma.aodn.org.au
+GEONETWORK_DB_PORT=5432
+GEONETWORK_DB_NAME=geonetwork
+GEONETWORK_DB_USERNAME=geonetwork

deploy/container/staging.env

@@ -0,0 +1,8 @@
+ES_HOST=ec2-3-25-64-248.ap-southeast-2.compute.amazonaws.com
+INDEXER_HOST=ec2-3-25-163-152.ap-southeast-2.compute.amazonaws.com
+INDEXER_PORT=8081
+GEONETWORK_DB_TYPE=postgres
+GEONETWORK_DB_HOST=stefan-db-rds-primary-evaluation.gamma.aodn.org.au
+GEONETWORK_DB_PORT=5432
+GEONETWORK_DB_NAME=geonetwork
+GEONETWORK_DB_USERNAME=geonetwork

deploy/github/development.env

@@ -1,18 +1,4 @@
-# general environment variables for Terragrunt
-ALB_PARAMETER_NAME=shared-alb-devops-sydney
-APP_NAME=geonetwork4
-AWS_ACCOUNT_ID=450356697252
-AWS_REGION=ap-southeast-2
+# variables required for GitHub Actions workflows
 ECR_REGISTRY=450356697252.dkr.ecr.ap-southeast-2.amazonaws.com
 ECR_REPOSITORY=geonetwork4
-ENVIRONMENT=tf-development
-
-# container definition variables
-ES_HOST=ec2-3-25-64-248.ap-southeast-2.compute.amazonaws.com
-INDEXER_HOST=ec2-3-25-163-152.ap-southeast-2.compute.amazonaws.com
-INDEXER_PORT=8081
-GEONETWORK_DB_TYPE=postgres
-GEONETWORK_DB_HOST=stefan-db-rds-primary-evaluation.gamma.aodn.org.au
-GEONETWORK_DB_PORT=5432
-GEONETWORK_DB_NAME=geonetwork
-GEONETWORK_DB_USERNAME=geonetwork
+ENVIRONMENT=development

deploy/github/production.env

@@ -1,19 +1,4 @@
-# general environment variables for Terragrunt
-ALB_PARAMETER_NAME=shared-alb-devops-sydney
-APP_HEALTH_CHECK="CMD-SHELL,uwsgi-is-ready --stats-socket /tmp/statsock > /dev/null 2>&1 || exit 1"
-APP_NAME=geonetwork4
-AWS_ACCOUNT_ID=450356697252
-AWS_REGION=ap-southeast-2
+# variables required for GitHub Actions workflows
 ECR_REGISTRY=450356697252.dkr.ecr.ap-southeast-2.amazonaws.com
 ECR_REPOSITORY=geonetwork4
 ENVIRONMENT=production
-
-# container definition variables
-ES_HOST=ec2-3-25-64-248.ap-southeast-2.compute.amazonaws.com
-INDEXER_HOST=ec2-3-25-163-152.ap-southeast-2.compute.amazonaws.com
-INDEXER_PORT=8081
-GEONETWORK_DB_TYPE=postgres
-GEONETWORK_DB_HOST=stefan-db-rds-primary-evaluation.gamma.aodn.org.au
-GEONETWORK_DB_PORT=5432
-GEONETWORK_DB_NAME=geonetwork
-GEONETWORK_DB_USERNAME=geonetwork

deploy/github/staging.env

@@ -1,19 +1,4 @@
-# general environment variables for Terragrunt
-ALB_PARAMETER_NAME=shared-alb-devops-sydney
-APP_HEALTH_CHECK="CMD-SHELL,uwsgi-is-ready --stats-socket /tmp/statsock > /dev/null 2>&1 || exit 1"
-APP_NAME=geonetwork4
-AWS_ACCOUNT_ID=450356697252
-AWS_REGION=ap-southeast-2
+# variables required for GitHub Actions workflows
 ECR_REGISTRY=450356697252.dkr.ecr.ap-southeast-2.amazonaws.com
 ECR_REPOSITORY=geonetwork4
 ENVIRONMENT=staging
-
-# container definition variables
-ES_HOST=ec2-3-25-64-248.ap-southeast-2.compute.amazonaws.com
-INDEXER_HOST=ec2-3-25-163-152.ap-southeast-2.compute.amazonaws.com
-INDEXER_PORT=8081
-GEONETWORK_DB_TYPE=postgres
-GEONETWORK_DB_HOST=stefan-db-rds-primary-evaluation.gamma.aodn.org.au
-GEONETWORK_DB_PORT=5432
-GEONETWORK_DB_NAME=geonetwork
-GEONETWORK_DB_USERNAME=geonetwork

deploy/tf/service.tf

@@ -1,13 +1,4 @@
 locals {
-  nginx_vars = {
-    app_host    = "127.0.0.1"
-    app_port    = var.app_port
-    listen_port = var.proxy_port
-  }
-
-  app_container_vars   = [for k, v in var.env_vars : { name = upper(k), value = v }]
-  nginx_container_vars = [for k, v in local.nginx_vars : { name = upper(k), value = v }]
-
   container_definitions = (
     var.nginx_proxy ?
     merge(local.app_container_definition, local.nginx_container_definition) :
@@ -28,7 +19,8 @@ locals {
       readonly_root_filesystem = false
       essential                = true
       memory_reservation       = 256
-      environment              = local.app_container_vars
+      environment              = var.env_vars
+      environment_files        = var.environment_files
       port_mappings = [
         {
           name          = var.app_container_name
@@ -46,6 +38,7 @@ locals {
       secrets = var.container_secrets
     }
   }
+
   nginx_container_definition = {
     nginx = {
       name  = "nginx"
@@ -56,7 +49,12 @@ locals {
       readonly_root_filesystem = false
       essential                = true
       memory_reservation       = 256
-      environment              = local.nginx_container_vars
+      environment = [
+        { name = "APP_HOST", value = "127.0.0.1" },
+        { name = "APP_PORT", value = var.app_port },
+        { name = "LISTEN_PORT", value = var.proxy_port }
+      ]
+      environment_files = []
       port_mappings = [
         {
           name          = "nginx"

deploy/tf/variables.tf

@@ -12,9 +12,9 @@ variable "app_container_name" {
 }
 
 variable "app_health_check" {
-  description = "The health check commmand to run on the docker container."
+  description = "The health check command to run on the docker container."
   type        = string
-  default     = null
+  default     = ""
 }
 
 variable "app_port" {
@@ -39,10 +39,22 @@ variable "cpu" {
 }
 
 variable "env_vars" {
-  description = "Map of key/pair values to pass to the container definition."
-  type        = map(any)
+  description = "List of key/pair values to pass to the container definition."
+  type = list(object({
+    value = string
+    type  = string
+  }))
+  default = []
 }
 
+variable "environment_files" {
+  description = "A list of files containing the environment variables to pass to a container"
+  type = list(object({
+    value = string
+    type  = string
+  }))
+  default = []
+}
 
 variable "image" {
   description = "The digest/tag of the docker image to pull from ECR"

deploy/tg/ecs/terragrunt.hcl

@@ -1,64 +1,21 @@
-#dependency "s3" {
-#  config_path  = "../s3"
-#  skip_outputs = true
-#}
+locals {
+  environment      = try(yamldecode(file("../..//vars/${local.environment_name}/environment.yaml")))
+  environment_name = get_env("ENVIRONMENT")
+  vars             = try(yamldecode(file("../..//vars/${local.environment_name}/variables.yaml")))
+}
 
 include "global" {
-  path   = "../global.hcl"
-  expose = true
+  path = "../global.hcl"
 }
 
-inputs = merge(local.override_vars, {
-  app_name         = get_env("APP_NAME")
-  app_health_check = get_env("APP_HEALTH_CHECK", "")
-  cluster_arn      = get_env("CLUSTER_ARN", "")
-  create_cluster   = get_env("CREATE_CLUSTER", true)
-  environment      = local.global.environment
-
-  # fetch the shared infrastructure parameter name
-  alb_parameter_name = get_env("ALB_PARAMETER_NAME")
-
-  # DNS hostnames to associate with the container
-  app_hostnames = split(",", get_env("APP_HOSTNAMES", local.default_hostname))
-
-  # container-specific environment variables
-  env_vars = local.env_vars
+inputs = merge(local.vars, {
+  app_name    = local.environment.app_name
+  environment = local.environment_name
 
   ecr_registry   = get_env("ECR_REGISTRY")
   ecr_repository = get_env("ECR_REPOSITORY")
-
-  iam_statements = local.iam_statements
 })
 
-locals {
-  global = include.global.locals
-
-  # container/task environment variables
-  default_env_vars = { for tuple in regexall("(.*?)=(.*)", file("../../container/default.env")) : tuple[0] => tuple[1] }
-
-  # get any overrides from the environment (e.g. GitHub deployment variables)
-  override_env_vars = {
-    for k, v in local.default_env_vars :
-    k => can(get_env(k)) ? get_env(k) : v
-  }
-
-  # remove empty values from the override map
-  env_vars = {
-    for k, v in local.override_env_vars : k => v if v != ""
-  }
-
-  default_hostname = join("-", [get_env("APP_NAME"), local.global.environment])
-
-  iam_statements = try(yamldecode(templatefile("../..//iam_statements/${local.global.environment}.yaml.tftpl",
-    {
-      aws_account = local.global.aws_account
-      aws_region  = local.global.aws_region
-      environment = local.global.environment
-  })), [])
-
-  override_vars = try(yamldecode(file("../../tf_vars/${local.global.environment}/variables.yaml")))
-}
-
 terraform {
   source = "../..//tf"
 }

deploy/tg/global.hcl

@@ -1,26 +1,24 @@
 locals {
-  aws_account  = get_env("AWS_ACCOUNT_ID")
-  aws_region   = get_env("AWS_REGION")
-  environment  = get_env("ENVIRONMENT")
-  project_name = get_env("APP_NAME")
-  state_bucket = "tfstate-${local.aws_account}-${local.aws_region}"
-  state_key    = "apps/${local.project_name}/${local.environment}/${basename(get_terragrunt_dir())}.tfstate"
+  environment      = try(yamldecode(file("..//vars/${local.environment_name}/environment.yaml")))
+  environment_name = get_env("ENVIRONMENT")
+  state_bucket     = "tfstate-${local.environment.aws_account_id}-${local.environment.aws_region}"
+  state_key        = "apps/${local.environment.app_name}/${local.environment_name}/${basename(get_terragrunt_dir())}.tfstate"
 }
 
 generate "providers" {
   path      = "providers.tf"
   if_exists = "overwrite_terragrunt"
   contents  = <<EOF
 provider "aws" {
-  region              = "${local.aws_region}"
-  allowed_account_ids = ["${local.aws_account}"]
+  region              = "${local.environment.aws_region}"
+  allowed_account_ids = ["${local.environment.aws_account_id}"]
   default_tags {
     tags = {
       "Environment" = "apps"
       "ManagedBy" = "Apps - ${local.state_bucket}/${local.state_key}"
       "Owner" = "Platform Engineering"
       "Project" = "AODN Applications"
-      "Repository" = "aodn/${local.project_name}"
+      "Repository" = "aodn/${local.environment.app_name}"
     }
   }
 }
@@ -36,7 +34,7 @@ remote_state {
   config = {
     bucket                      = local.state_bucket
     key                         = local.state_key
-    region                      = local.aws_region
+    region                      = local.environment.aws_region
     dynamodb_table              = local.state_bucket
     skip_credentials_validation = true
     skip_metadata_api_check     = true

deploy/vars/development/environment.yaml

@@ -0,0 +1,4 @@
+app_name: geonetwork4
+environment_name: development
+aws_account_id: 450356697252
+aws_region: ap-southeast-2