From 073fe6a28fb1cffeb4fe6a2e7d4220dba81bdda7 Mon Sep 17 00:00:00 2001 From: digorgonzola <29941279+digorgonzola@users.noreply.github.com> Date: Tue, 19 Dec 2023 17:35:30 +1100 Subject: [PATCH 01/51] update .gitignore with tf/tg rules --- .gitignore | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/.gitignore b/.gitignore index 91b83ad..89195d6 100644 --- a/.gitignore +++ b/.gitignore @@ -26,4 +26,12 @@ replay_pid* # Ingore local generated folders elasticdata/ gn4_data/ -.env \ No newline at end of file +.env + +# Terraform/Terragrunt directories and files +**/.terraform/* +**/.terragrunt-cache/ +*.tfstate +*.tfstate.* +terragrunt-debug.tfvars.json +terragrunt_rendered.json From e1b0282e53fccf5059328297e6dc0510905b7fc9 Mon Sep 17 00:00:00 2001 From: digorgonzola <29941279+digorgonzola@users.noreply.github.com> Date: Tue, 19 Dec 2023 17:35:53 +1100 Subject: [PATCH 02/51] add .pre-commit-config.yaml --- .pre-commit-config.yaml | 25 +++++++++++++++++++++++++ 1 file changed, 25 insertions(+) create mode 100644 .pre-commit-config.yaml diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml new file mode 100644 index 0000000..f953723 --- /dev/null +++ b/.pre-commit-config.yaml @@ -0,0 +1,25 @@ +# See https://pre-commit.com for more information +# See https://pre-commit.com/hooks.html for more hooks +repos: + - repo: https://github.com/pre-commit/pre-commit-hooks + rev: v4.4.0 + hooks: + - id: trailing-whitespace + - id: end-of-file-fixer + - id: check-yaml + - id: check-added-large-files + # Security + - id: detect-private-key + - repo: https://github.com/gitleaks/gitleaks + rev: v8.18.0 + hooks: + - id: gitleaks + - repo: https://github.com/antonbabenko/pre-commit-terraform + rev: v1.83.4 + hooks: + - id: terraform_fmt + - id: terragrunt_fmt + - id: terraform_validate + args: + - --tf-init-args=-backend=false + - id: terragrunt_fmt From d1ed20d53d8c9463c4a9004d2db60b5157107aeb Mon Sep 17 00:00:00 2001 From: digorgonzola <29941279+digorgonzola@users.noreply.github.com> Date: Tue, 19 Dec 2023 17:36:57 +1100 Subject: [PATCH 03/51] formatting fixes for existing code --- README.md | 6 +++--- pom.xml | 2 +- src/main/resources/application.properties | 2 +- src/main/resources/schema/gn4-api.json | 2 +- startEsLocal.sh | 1 - 5 files changed, 6 insertions(+), 7 deletions(-) diff --git a/README.md b/README.md index 7f732dd..a2a31db 100644 --- a/README.md +++ b/README.md @@ -4,7 +4,7 @@ This repo is used to customize a GeoNetwork4 to be used by AODN. The key customi build a jar file with the code we want to add and insert it use COPY function to copy it to the lib folder during the Dockerfile build. -The jar file contains a hack where we name a @Configuration class with the same package as +The jar file contains a hack where we name a @Configuration class with the same package as geonetwork4 base package (org.fao.geonet), so that the initial component-scan will pick up this class. From there we add additional component-scan to our custom classes. This avoided the need to alter the xml like what we did before plus we are using a Docker base image of GeoNetwork4. @@ -29,6 +29,6 @@ Once you have the json, you can generate code like the one here in Java to acces ## Use of S3 -You can see a config file related to S3, however we do not use it because after experiment it, it +You can see a config file related to S3, however we do not use it because after experiment it, it didn't support well as the GN4 will issue warning on file not found with relative folder name. The -code is just keep as a record. +code is just keep as a record. diff --git a/pom.xml b/pom.xml index 448b1d3..b2ec6a1 100644 --- a/pom.xml +++ b/pom.xml @@ -179,4 +179,4 @@ - \ No newline at end of file + diff --git a/src/main/resources/application.properties b/src/main/resources/application.properties index 00a396b..7cb1cc4 100644 --- a/src/main/resources/application.properties +++ b/src/main/resources/application.properties @@ -2,4 +2,4 @@ aodn.geonetwork4.esIndexer.protocol=http aodn.geonetwork4.esIndexer.host=${INDEXER_HOST:localhost} aodn.geonetwork4.esIndexer.port=${INDEXER_PORT:80} aodn.geonetwork4.esIndexer.apikey=${INDEXER_APIKEY} -aodn.geonetwork4.esIndexer.urlIndex=${aodn.geonetwork4.esIndexer.protocol}://${aodn.geonetwork4.esIndexer.host}:${aodn.geonetwork4.esIndexer.port}/api/v1/indexer/index/{uuid} \ No newline at end of file +aodn.geonetwork4.esIndexer.urlIndex=${aodn.geonetwork4.esIndexer.protocol}://${aodn.geonetwork4.esIndexer.host}:${aodn.geonetwork4.esIndexer.port}/api/v1/indexer/index/{uuid} diff --git a/src/main/resources/schema/gn4-api.json b/src/main/resources/schema/gn4-api.json index a176751..c15e17b 100644 --- a/src/main/resources/schema/gn4-api.json +++ b/src/main/resources/schema/gn4-api.json @@ -22473,4 +22473,4 @@ } } } -} \ No newline at end of file +} diff --git a/startEsLocal.sh b/startEsLocal.sh index 92b505d..99a8326 100755 --- a/startEsLocal.sh +++ b/startEsLocal.sh @@ -31,4 +31,3 @@ fi # If you run in EC2, you need to change the docker-es-compose.yml localhost in cert to the hostname of EC2 and install docker # docker-compose and sudo chmod 666 /run/docker.sock docker-compose -f docker-es-compose.yml up --detach - From 162806a29e1c3e8bd23bf7c9fbce61e09b692896 Mon Sep 17 00:00:00 2001 From: digorgonzola <29941279+digorgonzola@users.noreply.github.com> Date: Tue, 19 Dec 2023 17:38:53 +1100 Subject: [PATCH 04/51] initial commit for deployment code --- deploy/Dockerfile | 26 +++ deploy/README.md | 60 +++++++ deploy/container/default.env | 10 ++ deploy/docker-compose.yml | 18 +++ deploy/github/development.env | 20 +++ deploy/github/production.env | 21 +++ deploy/github/staging.env | 21 +++ deploy/iam_statements/development.yaml.tftpl | 14 ++ deploy/iam_statements/production.yaml.tftpl | 0 deploy/iam_statements/staging.yaml.tftpl | 0 deploy/tf/alb.tf | 41 +++++ deploy/tf/aws_details.tf | 2 + deploy/tf/cluster.tf | 26 +++ deploy/tf/get-parameters.tf | 62 +++++++ deploy/tf/main.tf | 0 deploy/tf/outputs.tf | 7 + deploy/tf/service.tf | 161 +++++++++++++++++++ deploy/tf/variables.tf | 100 ++++++++++++ deploy/tf/versions.tf | 9 ++ deploy/tg/ecs/.terraform.lock.hcl | 46 ++++++ deploy/tg/ecs/terragrunt.hcl | 62 +++++++ deploy/tg/global.hcl | 47 ++++++ 22 files changed, 753 insertions(+) create mode 100644 deploy/Dockerfile create mode 100644 deploy/README.md create mode 100644 deploy/container/default.env create mode 100644 deploy/docker-compose.yml create mode 100644 deploy/github/development.env create mode 100644 deploy/github/production.env create mode 100644 deploy/github/staging.env create mode 100644 deploy/iam_statements/development.yaml.tftpl create mode 100644 deploy/iam_statements/production.yaml.tftpl create mode 100644 deploy/iam_statements/staging.yaml.tftpl create mode 100644 deploy/tf/alb.tf create mode 100644 deploy/tf/aws_details.tf create mode 100644 deploy/tf/cluster.tf create mode 100644 deploy/tf/get-parameters.tf create mode 100644 deploy/tf/main.tf create mode 100644 deploy/tf/outputs.tf create mode 100644 deploy/tf/service.tf create mode 100644 deploy/tf/variables.tf create mode 100644 deploy/tf/versions.tf create mode 100644 deploy/tg/ecs/.terraform.lock.hcl create mode 100644 deploy/tg/ecs/terragrunt.hcl create mode 100644 deploy/tg/global.hcl diff --git a/deploy/Dockerfile b/deploy/Dockerfile new file mode 100644 index 0000000..694ea53 --- /dev/null +++ b/deploy/Dockerfile @@ -0,0 +1,26 @@ +ARG TF_VERSION=1.5.7 + +FROM hashicorp/terraform:$TF_VERSION + +LABEL maintainer="devops@aodn.org.au" + +ARG TG_VERSION=0.54.0 +ARG USER_ID=1000 +ARG GROUP_ID=1000 +ARG USERNAME=appdeploy + +RUN wget https://github.com/gruntwork-io/terragrunt/releases/download/v${TG_VERSION}/terragrunt_linux_amd64 \ + && mv terragrunt_linux_amd64 terragrunt \ + && chmod +x terragrunt \ + && mv terragrunt /usr/local/bin/terragrunt + +RUN apk add --no-cache aws-cli + +RUN echo 'alias tf="terraform"' >> /etc/profile.d/custom_aliases.sh \ + && echo 'alias tg="terragrunt"' >> /etc/profile.d/custom_aliases.sh + +ENV ENV="/etc/profile" + +RUN adduser -D -u $USER_ID -h /home/$USERNAME $USERNAME + +USER $USERNAME:$USERNAME diff --git a/deploy/README.md b/deploy/README.md new file mode 100644 index 0000000..53fa71b --- /dev/null +++ b/deploy/README.md @@ -0,0 +1,60 @@ +# Deployments +Deployment of this application uses [Github Deployment Environments](https://docs.github.com/en/actions/deployment/targeting-different-environments/using-environments-for-deployment). + +A successful deployment relies on the correct variables being defined in order to deploy to the correct AWS account etc. + +### Managing Environments +You can view the current environment settings by visiting https://github.com/aodn/sample-django-app/settings/environments. + +You can view the currently defined variables there or from the cli using: +```bash +gh variable list -R aodn/sample-django-app -e staging +``` + +### Updating Variables +Manually updating vars can be tedious and error-prone. Instead, you may define the variables you need as a .env file and push these values: +```bash +gh variable set -R aodn/sample-django-app -e staging -f staging.env +``` + +### Deploying From the CLI +The Terragrunt module for this application depends on variables being present and will fail if they are not. + +To test locally, you may want to populate these into your local environment using the following command: +```bash +set -a; source ./github/staging.env; set +a +terragrunt plan +terragrunt apply +``` + +### Using Docker +A Dockerfile is provided to simplify local deployment, removing the need to install the required binaries on the local system. + +N.B. The Dockerfile does assume a valid AWS CLI configuration. + +### Example deployment +Modify the environment variables in `dev.env`: +```text +ALB_PARAMETER_NAME=shared-alb-dev-sydney +APP_NAME=sample-django-app-mybranch +AWS_ACCOUNT_ID=123456789012 +AWS_REGION=ap-southeast-2 +ECR_PARAMETER_NAME=api +ECR_REGISTRY=123456789012.dkr.ecr.ap-southeast-2.amazonaws.com +ECR_REPOSITORY=api +ENVIRONMENT=mydev-stack +RDS_PARAMETER_NAME=db01/primary/development +``` + +```bash +export AWS_PROFILE=myprofile +cd deploy +docker-compose -f docker-compose.yml run terragrunt +``` + +On the container run the following: +```bash +set -a; source ./github/dev.env; set +a +TF_VAR_image=latest terragrunt plan -out=tf.plan +TF_VAR_image=latest terragrunt apply -auto-approve tf.plan +``` diff --git a/deploy/container/default.env b/deploy/container/default.env new file mode 100644 index 0000000..07044d6 --- /dev/null +++ b/deploy/container/default.env @@ -0,0 +1,10 @@ +ES_HOST=ec2-3-25-64-248.ap-southeast-2.compute.amazonaws.com +INDEXER_HOST=ec2-3-25-163-152.ap-southeast-2.compute.amazonaws.com +INDEXER_PORT=8081 +INDEXER_APIKEY= +GEONETWORK_DB_TYPE=postgres +GEONETWORK_DB_HOST=db +GEONETWORK_DB_PORT=5432 +GEONETWORK_DB_NAME=geonetwork +GEONETWORK_DB_USERNAME=geonetwork +GEONETWORK_DB_PASSWORD= diff --git a/deploy/docker-compose.yml b/deploy/docker-compose.yml new file mode 100644 index 0000000..ebc90ea --- /dev/null +++ b/deploy/docker-compose.yml @@ -0,0 +1,18 @@ +version: '3.7' + +services: + terragrunt: + build: + context: . + dockerfile: Dockerfile + args: + TF_VERSION: 1.5.7 + TG_VERSION: 0.54.0 + image: appdeploy:latest + volumes: + - .:/deploy + - ${HOME}/.aws:/home/appdeploy/.aws + working_dir: /deploy/tg + environment: + - AWS_PROFILE=${AWS_PROFILE} + entrypoint: ["sh"] diff --git a/deploy/github/development.env b/deploy/github/development.env new file mode 100644 index 0000000..51f2eb9 --- /dev/null +++ b/deploy/github/development.env @@ -0,0 +1,20 @@ +# general environment variables for Terragrunt +ALB_PARAMETER_NAME=shared-alb-devops-sydney +APP_NAME=geonetwork4 +AWS_ACCOUNT_ID=450356697252 +AWS_REGION=ap-southeast-2 +ECR_REGISTRY=450356697252.dkr.ecr.ap-southeast-2.amazonaws.com +ECR_REPOSITORY=geonetwork4 +ENVIRONMENT=development +TF_VAR_app_port=80 +TF_VAR_nginx_proxy=false + +# container definition variables +ES_HOST=ec2-3-25-64-248.ap-southeast-2.compute.amazonaws.com +INDEXER_HOST=ec2-3-25-163-152.ap-southeast-2.compute.amazonaws.com +INDEXER_PORT=8081 +GEONETWORK_DB_TYPE=postgres +GEONETWORK_DB_HOST=stefan-db-rds-primary-evaluation.gamma.aodn.org.au +GEONETWORK_DB_PORT=5432 +GEONETWORK_DB_NAME=geonetwork +GEONETWORK_DB_USERNAME=geonetwork diff --git a/deploy/github/production.env b/deploy/github/production.env new file mode 100644 index 0000000..b98434a --- /dev/null +++ b/deploy/github/production.env @@ -0,0 +1,21 @@ +# general environment variables for Terragrunt +ALB_PARAMETER_NAME=shared-alb-devops-sydney +APP_HEALTH_CHECK="CMD-SHELL,uwsgi-is-ready --stats-socket /tmp/statsock > /dev/null 2>&1 || exit 1" +APP_NAME=geonetwork4 +AWS_ACCOUNT_ID=450356697252 +AWS_REGION=ap-southeast-2 +ECR_REGISTRY=450356697252.dkr.ecr.ap-southeast-2.amazonaws.com +ECR_REPOSITORY=geonetwork4 +ENVIRONMENT=production +TF_VAR_app_port=80 +TF_VAR_nginx_proxy=false + +# container definition variables +ES_HOST=ec2-3-25-64-248.ap-southeast-2.compute.amazonaws.com +INDEXER_HOST=ec2-3-25-163-152.ap-southeast-2.compute.amazonaws.com +INDEXER_PORT=8081 +GEONETWORK_DB_TYPE=postgres +GEONETWORK_DB_HOST=stefan-db-rds-primary-evaluation.gamma.aodn.org.au +GEONETWORK_DB_PORT=5432 +GEONETWORK_DB_NAME=geonetwork +GEONETWORK_DB_USERNAME=geonetwork diff --git a/deploy/github/staging.env b/deploy/github/staging.env new file mode 100644 index 0000000..8e1b2a3 --- /dev/null +++ b/deploy/github/staging.env @@ -0,0 +1,21 @@ +# general environment variables for Terragrunt +ALB_PARAMETER_NAME=shared-alb-devops-sydney +APP_HEALTH_CHECK="CMD-SHELL,uwsgi-is-ready --stats-socket /tmp/statsock > /dev/null 2>&1 || exit 1" +APP_NAME=geonetwork4 +AWS_ACCOUNT_ID=450356697252 +AWS_REGION=ap-southeast-2 +ECR_REGISTRY=450356697252.dkr.ecr.ap-southeast-2.amazonaws.com +ECR_REPOSITORY=geonetwork4 +ENVIRONMENT=staging +TF_VAR_app_port=80 +TF_VAR_nginx_proxy=false + +# container definition variables +ES_HOST=ec2-3-25-64-248.ap-southeast-2.compute.amazonaws.com +INDEXER_HOST=ec2-3-25-163-152.ap-southeast-2.compute.amazonaws.com +INDEXER_PORT=8081 +GEONETWORK_DB_TYPE=postgres +GEONETWORK_DB_HOST=stefan-db-rds-primary-evaluation.gamma.aodn.org.au +GEONETWORK_DB_PORT=5432 +GEONETWORK_DB_NAME=geonetwork +GEONETWORK_DB_USERNAME=geonetwork diff --git a/deploy/iam_statements/development.yaml.tftpl b/deploy/iam_statements/development.yaml.tftpl new file mode 100644 index 0000000..b4075ea --- /dev/null +++ b/deploy/iam_statements/development.yaml.tftpl @@ -0,0 +1,14 @@ +#- actions: +# - s3:PutObject +# - s3:GetObjectAcl +# - s3:GetObject +# - s3:ListBucket +# - s3:DeleteObject +# - s3:PutObjectAcl +# resources: +# - arn:aws:s3:::sample-bucket-${environment}-${aws_account} +# - arn:aws:s3:::sample-bucket-${environment}-${aws_account}/* +#- actions: +# - secretsmanager:GetSecretValue +# resources: +# - arn:aws:secretsmanager:${aws_region}:${aws_account}:secret:/rds/* diff --git a/deploy/iam_statements/production.yaml.tftpl b/deploy/iam_statements/production.yaml.tftpl new file mode 100644 index 0000000..e69de29 diff --git a/deploy/iam_statements/staging.yaml.tftpl b/deploy/iam_statements/staging.yaml.tftpl new file mode 100644 index 0000000..e69de29 diff --git a/deploy/tf/alb.tf b/deploy/tf/alb.tf new file mode 100644 index 0000000..e88b353 --- /dev/null +++ b/deploy/tf/alb.tf @@ -0,0 +1,41 @@ +resource "aws_lb_target_group" "app" { + name = "${var.app_name}-${var.environment}" + port = var.nginx_proxy ? var.proxy_port : var.app_port + protocol = "HTTP" + target_type = "ip" + vpc_id = local.vpc_id + + health_check { + enabled = true + path = "/health" + } +} + +resource "aws_route53_record" "app" { + for_each = toset(var.app_hostnames) + zone_id = local.domain_zone_id + name = each.value + type = "A" + + alias { + name = local.alb_dns_name + zone_id = local.alb_zone_id + evaluate_target_health = true + } +} + +resource "aws_lb_listener_rule" "app_fgate" { + for_each = toset(var.app_hostnames) + listener_arn = local.alb_https_listener_arn + + action { + type = "forward" + target_group_arn = aws_lb_target_group.app.arn + } + + condition { + host_header { + values = [aws_route53_record.app[each.value].fqdn] + } + } +} diff --git a/deploy/tf/aws_details.tf b/deploy/tf/aws_details.tf new file mode 100644 index 0000000..038d1e2 --- /dev/null +++ b/deploy/tf/aws_details.tf @@ -0,0 +1,2 @@ +data "aws_caller_identity" "current" {} +data "aws_region" "current" {} diff --git a/deploy/tf/cluster.tf b/deploy/tf/cluster.tf new file mode 100644 index 0000000..3a7edaa --- /dev/null +++ b/deploy/tf/cluster.tf @@ -0,0 +1,26 @@ +module "cluster" { + source = "terraform-aws-modules/ecs/aws//modules/cluster" + version = "~> 5.7.0" + + create = var.create_cluster ? true : false + + # Cluster Configuration + cluster_name = "${var.app_name}-${var.environment}" + cluster_configuration = { + name = "containerInsights" + value = "enabled" + } + create_task_exec_iam_role = true + fargate_capacity_providers = { + FARGATE = { + default_capacity_provider_strategy = { + weight = 50 + } + } + FARGATE_SPOT = { + default_capacity_provider_strategy = { + weight = 50 + } + } + } +} diff --git a/deploy/tf/get-parameters.tf b/deploy/tf/get-parameters.tf new file mode 100644 index 0000000..d80de7c --- /dev/null +++ b/deploy/tf/get-parameters.tf @@ -0,0 +1,62 @@ +locals { + # alb values + alb_dns_name = nonsensitive(data.aws_ssm_parameter.alb_dns_name.value) + alb_https_listener_arn = nonsensitive(data.aws_ssm_parameter.alb_https_listener_arn.value) + alb_zone_id = nonsensitive(data.aws_ssm_parameter.alb_zone_id.value) + + # core values + vpc_id = nonsensitive(data.aws_ssm_parameter.vpc_id.value) + vpc_cidr = nonsensitive(data.aws_ssm_parameter.vpc_cidr.value) + domain_name = nonsensitive(data.aws_ssm_parameter.zonename.value) + domain_zone_id = nonsensitive(data.aws_ssm_parameter.zoneid.value) + public_subnets = split(",", nonsensitive(data.aws_ssm_parameter.public_subnets.value)) + public_subnet_cidrs = nonsensitive(data.aws_ssm_parameter.public_subnet_cidrs.value) + private_subnets = split(",", nonsensitive(data.aws_ssm_parameter.private_subnets.value)) + private_subnet_cidrs = nonsensitive(data.aws_ssm_parameter.private_subnet_cidrs.value) +} + +# alb parameters +data "aws_ssm_parameter" "alb_dns_name" { + name = "/apps/alb/${var.alb_parameter_name}/alb_dns_name" +} + +data "aws_ssm_parameter" "alb_https_listener_arn" { + name = "/apps/alb/${var.alb_parameter_name}/alb_https_listener_arn" +} + +data "aws_ssm_parameter" "alb_zone_id" { + name = "/apps/alb/${var.alb_parameter_name}/alb_zone_id" +} + +# core parameters +data "aws_ssm_parameter" "vpc_id" { + name = "/core/vpc_id" +} + +data "aws_ssm_parameter" "vpc_cidr" { + name = "/core/vpc_cidr" +} + +data "aws_ssm_parameter" "public_subnets" { + name = "/core/subnets_public" +} + +data "aws_ssm_parameter" "public_subnet_cidrs" { + name = "/core/subnets_public_cidr" +} + +data "aws_ssm_parameter" "private_subnets" { + name = "/core/subnets_private" +} + +data "aws_ssm_parameter" "private_subnet_cidrs" { + name = "/core/subnets_private_cidr" +} + +data "aws_ssm_parameter" "zonename" { + name = "/core/zone_domain" +} + +data "aws_ssm_parameter" "zoneid" { + name = "/core/zone_id" +} diff --git a/deploy/tf/main.tf b/deploy/tf/main.tf new file mode 100644 index 0000000..e69de29 diff --git a/deploy/tf/outputs.tf b/deploy/tf/outputs.tf new file mode 100644 index 0000000..20f61bb --- /dev/null +++ b/deploy/tf/outputs.tf @@ -0,0 +1,7 @@ +output "cluster" { + value = module.cluster +} + +output "service" { + value = module.service +} diff --git a/deploy/tf/service.tf b/deploy/tf/service.tf new file mode 100644 index 0000000..6a1b9a0 --- /dev/null +++ b/deploy/tf/service.tf @@ -0,0 +1,161 @@ +locals { + nginx_vars = { + app_host = "127.0.0.1" + app_port = var.app_port + listen_port = var.proxy_port + } + + app_container_vars = [for k, v in var.env_vars : { name = upper(k), value = v }] + nginx_container_vars = [for k, v in local.nginx_vars : { name = upper(k), value = v }] + + container_definitions = ( + var.nginx_proxy ? + merge(local.app_container_definition, local.nginx_container_definition) : + local.app_container_definition + ) + + app_container_definition = { + app = { + name = var.app_container_name + image = ( + startswith(var.image, "sha256") ? + "${var.ecr_registry}/${var.ecr_repository}@${var.image}" : + "${var.ecr_registry}/${var.ecr_repository}:${var.image}" + ) + health_check = { + command = length(var.app_health_check) > 0 ? split(",", var.app_health_check) : [] + } + readonly_root_filesystem = false + essential = true + memory_reservation = 256 + environment = local.app_container_vars + port_mappings = [ + { + name = var.app_container_name + containerPort = var.app_port + hostPort = var.app_port + } + ] + mount_points = [ + { + readOnly = false + containerPath = "/vol/web" + sourceVolume = "static" + } + ] + } + } + nginx_container_definition = { + nginx = { + name = "nginx" + image = "${var.ecr_registry}/nginx-proxy:latest" + health_check = { + command = ["CMD-SHELL", "curl -so /dev/null http://localhost/health || exit 1"] + } + readonly_root_filesystem = false + essential = true + memory_reservation = 256 + environment = local.nginx_container_vars + port_mappings = [ + { + name = "nginx" + containerPort = var.proxy_port + hostPort = var.proxy_port + } + ] + mount_points = [ + { + readOnly = false + containerPath = "/vol/static" + sourceVolume = "static" + } + ] + } + } +} + +resource "null_resource" "cluster_arn_precondition_check" { + lifecycle { + precondition { + condition = (var.create_cluster == false && var.cluster_arn != "" || var.create_cluster && var.cluster_arn == "") + error_message = "The cluster ARN must be provided if 'create_cluster' is false. If you mean to have this module create the cluster, set 'create_cluster' to true." + } + } +} + +module "service" { + source = "terraform-aws-modules/ecs/aws//modules/service" + version = "~> 5.7.0" + + name = "${var.app_name}-${var.environment}" + cluster_arn = var.create_cluster ? module.cluster.arn : var.cluster_arn + capacity_provider_strategy = { + env_strategy = { + base = 0 + capacity_provider = var.environment == "production" ? "FARGATE" : "FARGATE_SPOT" + weight = 100 + } + } + + # allow ECS exec commands on containers (e.g. to get a shell session) + enable_execute_command = true + + # resources + cpu = var.cpu + memory = var.memory + + # do not force a new deployment unless the image digest has changed + force_new_deployment = false + + # wait for service to reach steady state + wait_for_steady_state = true + + # wait for the task to reach steady state + wait_until_stable = true + + # Container definition(s) + container_definitions = local.container_definitions + + deployment_circuit_breaker = { + enable = true + rollback = true + } + + load_balancer = { + service = { + target_group_arn = aws_lb_target_group.app.arn + container_name = var.nginx_proxy ? "nginx" : "app" + container_port = var.nginx_proxy ? var.proxy_port : var.app_port + } + } + + subnet_ids = local.private_subnets + + security_group_rules = { + ingress_vpc = { + type = "ingress" + from_port = var.nginx_proxy ? var.proxy_port : var.app_port + to_port = var.nginx_proxy ? var.proxy_port : var.app_port + protocol = "tcp" + cidr_blocks = [local.vpc_cidr] + } + egress_all = { + type = "egress" + from_port = 0 + to_port = 0 + protocol = "-1" + cidr_blocks = ["0.0.0.0/0"] + } + } + + tasks_iam_role_statements = var.iam_statements + + timeouts = { + create = "10m" + update = "10m" + } + + volume = { + static = {} + } +} diff --git a/deploy/tf/variables.tf b/deploy/tf/variables.tf new file mode 100644 index 0000000..614e91f --- /dev/null +++ b/deploy/tf/variables.tf @@ -0,0 +1,100 @@ +variable "alb_parameter_name" { + description = "The parameter name to derive the ALB details from." + type = string +} + +variable "app_container_name" { + description = "The name of the primary application container" + type = string + default = "app" +} + +variable "app_health_check" { + description = "The health check commmand to run on the docker container." + type = string + default = null +} + +variable "app_hostnames" { + description = "Hostnames to associate with the application" + type = list(string) +} + +variable "app_name" { + description = "The name of the application e.g. sample-django-app" + type = string +} + +variable "app_port" { + description = "The port to expose to the nginx proxy on the application container." + type = number + default = 9000 +} + +variable "cluster_arn" { + description = "ARN of the existing cluster to deploy the service/tasks to." + type = string + default = "" +} + +variable "cpu" { + description = "The CPU capacity to allocate to the task." + type = number + default = 512 +} + +variable "create_cluster" { + description = "Whether or not to create a separate cluster for this deployment. If false, the name of an existing cluster must be provided." + type = bool + default = true +} + +variable "ecr_registry" { + description = "The registry to pull docker images from." + type = string +} + +variable "ecr_repository" { + description = "The repository to pull the image from." + type = string +} + +variable "env_vars" { + description = "Map of key/pair values to pass to the container definition." + type = map(any) +} + +variable "environment" { + description = "Environment name to prepend/append to resource names" + type = string +} + +variable "iam_statements" { + description = "List of IAM statements to attach to the task role" + type = any + default = [] +} + +variable "image" { + description = "The digest/tag of the docker image to pull from ECR" + type = string + default = "latest" +} + +variable "memory" { + description = "The CPU capacity to allocate to the task." + type = number + default = 1024 +} + +variable "nginx_proxy" { + description = "Whether or not to side-load an nginx container in the task definition" + type = bool + default = true +} + +variable "proxy_port" { + description = "The port to expose to the load balancer on the container" + type = number + default = 80 +} diff --git a/deploy/tf/versions.tf b/deploy/tf/versions.tf new file mode 100644 index 0000000..91041e5 --- /dev/null +++ b/deploy/tf/versions.tf @@ -0,0 +1,9 @@ +terraform { + required_version = "~> 1.5.0" + required_providers { + aws = { + source = "hashicorp/aws" + version = "~> 5.12" + } + } +} diff --git a/deploy/tg/ecs/.terraform.lock.hcl b/deploy/tg/ecs/.terraform.lock.hcl new file mode 100644 index 0000000..fad1fd7 --- /dev/null +++ b/deploy/tg/ecs/.terraform.lock.hcl @@ -0,0 +1,46 @@ +# This file is maintained automatically by "terraform init". +# Manual edits may be lost in future updates. + +provider "registry.terraform.io/hashicorp/aws" { + version = "5.29.0" + constraints = ">= 4.9.0, >= 4.66.1, ~> 5.12" + hashes = [ + "h1:+4qYlyPaktjZTFP9UbpEaz55jfwWapdibViCPoRFf+s=", + "h1:6Wb/0B+dNeOBP5KXvlHOmZyX7O6CDEfR+MrbxWI3wfo=", + "h1:SyiKAX/D3ZE9My7P03DrRMf65pNnfSDQXPb0g11lCS0=", + "zh:0453c1c64e51cd7050ce46d9280a0195b9073592508077ebf1a1c45f7026f3f5", + "zh:3ee87d1a2870b61fdcc80f3f96b669dbcc8171aadb821bec0e1fa0e6fb9595b6", + "zh:423c0304eba345167cc37dcd300712f24f03fe4de8eecc15edb0d4f88b29ec79", + "zh:6816ce0ed702263297a8e02467bb712c509a9f6e4f132a152a10f1cc19191a81", + "zh:6feb8a0aedabd778216238e72273f5c2ee86d8841acc3fb3dc9d8014a2bbdc51", + "zh:709ccdc8b37f975d422e7955814671548887613931e234e06249da629b0f2f95", + "zh:76c55744020dbdafea25be634f8ac37c1e371f8c397f73bd89bc270d00ee0834", + "zh:7e48d6fc488b9dbe2fd4bebefa1b485d04da38b11a6799f8cba178173b7f8782", + "zh:951d7ef2adbfb96b1d3e9c4780b2ab4375caf9c6b522a2d023c02ff0698d8e2a", + "zh:9b12af85486a96aedd8d7984b0ff811a4b42e3d88dad1a3fb4c0b580d04fa425", + "zh:b0bf5974bc1a7d2ce3f3a9a31a8238ad15ad02211f1e84c54832541ec4bd5d10", + "zh:cc56d4ab9bcbee95f73dbe90f11d4ff7299b835dddf2b30cfda526a2cccd0f9f", + "zh:cfe3a4394f2f7044e03bb63f4fb9c691926607c6784417ac9c0724943da60d09", + "zh:d6f82e13f33f70de8df480287b5a961ced5606f041d1c589f706b112f68db890", + "zh:fb7be5bcff62d0ca9edd4a1bee4d2ed16e9428e3f9eff3ea4d898ecb234505a3", + ] +} + +provider "registry.terraform.io/hashicorp/null" { + version = "3.2.2" + hashes = [ + "h1:IMVAUHKoydFrlPrl9OzasDnw/8ntZFerCC9iXw1rXQY=", + "zh:3248aae6a2198f3ec8394218d05bd5e42be59f43a3a7c0b71c66ec0df08b69e7", + "zh:32b1aaa1c3013d33c245493f4a65465eab9436b454d250102729321a44c8ab9a", + "zh:38eff7e470acb48f66380a73a5c7cdd76cc9b9c9ba9a7249c7991488abe22fe3", + "zh:4c2f1faee67af104f5f9e711c4574ff4d298afaa8a420680b0cb55d7bbc65606", + "zh:544b33b757c0b954dbb87db83a5ad921edd61f02f1dc86c6186a5ea86465b546", + "zh:696cf785090e1e8cf1587499516b0494f47413b43cb99877ad97f5d0de3dc539", + "zh:6e301f34757b5d265ae44467d95306d61bef5e41930be1365f5a8dcf80f59452", + "zh:78d5eefdd9e494defcb3c68d282b8f96630502cac21d1ea161f53cfe9bb483b3", + "zh:913a929070c819e59e94bb37a2a253c228f83921136ff4a7aa1a178c7cce5422", + "zh:aa9015926cd152425dbf86d1abdbc74bfe0e1ba3d26b3db35051d7b9ca9f72ae", + "zh:bb04798b016e1e1d49bcc76d62c53b56c88c63d6f2dfe38821afef17c416a0e1", + "zh:c23084e1b23577de22603cff752e59128d83cfecc2e6819edadd8cf7a10af11e", + ] +} diff --git a/deploy/tg/ecs/terragrunt.hcl b/deploy/tg/ecs/terragrunt.hcl new file mode 100644 index 0000000..44cfe53 --- /dev/null +++ b/deploy/tg/ecs/terragrunt.hcl @@ -0,0 +1,62 @@ +#dependency "s3" { +# config_path = "../s3" +# skip_outputs = true +#} + +include "global" { + path = "../global.hcl" + expose = true +} + +inputs = { + app_name = get_env("APP_NAME") + app_health_check = get_env("APP_HEALTH_CHECK", "") + cluster_arn = get_env("CLUSTER_ARN", "") + create_cluster = get_env("CREATE_CLUSTER", true) + environment = local.global.environment + + # fetch the shared infrastructure parameter name + alb_parameter_name = get_env("ALB_PARAMETER_NAME") + + # DNS hostnames to associate with the container + app_hostnames = split(",", get_env("APP_HOSTNAMES", local.default_hostname)) + + # container-specific environment variables + env_vars = local.env_vars + + ecr_registry = get_env("ECR_REGISTRY") + ecr_repository = get_env("ECR_REPOSITORY") + + iam_statements = local.iam_statements +} + +locals { + global = include.global.locals + + # container/task environment variables + default_env_vars = { for tuple in regexall("(.*?)=(.*)", file("../../container/default.env")) : tuple[0] => tuple[1] } + + # get any overrides from the environment (e.g. GitHub deployment variables) + override_env_vars = { + for k, v in local.default_env_vars : + k => can(get_env(k)) ? get_env(k) : v + } + + # remove empty values from the override map + env_vars = { + for k, v in local.override_env_vars : k => v if v != "" + } + + default_hostname = join("-", [get_env("APP_NAME"), local.global.environment]) + + iam_statements = try(yamldecode(templatefile("../..//iam_statements/${local.global.environment}.yaml.tftpl", + { + aws_account = local.global.aws_account + aws_region = local.global.aws_region + environment = local.global.environment + })), []) +} + +terraform { + source = "../..//tf" +} diff --git a/deploy/tg/global.hcl b/deploy/tg/global.hcl new file mode 100644 index 0000000..19e0292 --- /dev/null +++ b/deploy/tg/global.hcl @@ -0,0 +1,47 @@ +locals { + aws_account = get_env("AWS_ACCOUNT_ID") + aws_region = get_env("AWS_REGION") + environment = get_env("ENVIRONMENT") + project_name = get_env("APP_NAME") + state_bucket = "tfstate-${local.aws_account}-${local.aws_region}" + state_key = "apps/${local.project_name}/${local.environment}/${basename(get_terragrunt_dir())}.tfstate" +} + +generate "providers" { + path = "providers.tf" + if_exists = "overwrite_terragrunt" + contents = < Date: Tue, 19 Dec 2023 17:39:13 +1100 Subject: [PATCH 05/51] init commit for github workflows --- .github/workflows/deploy-development.yml | 141 +++++++++++++++++++++++ .github/workflows/deploy-production.yml | 132 +++++++++++++++++++++ .github/workflows/deploy-staging.yml | 114 ++++++++++++++++++ .github/workflows/pre-commit.yml | 32 +++++ 4 files changed, 419 insertions(+) create mode 100644 .github/workflows/deploy-development.yml create mode 100644 .github/workflows/deploy-production.yml create mode 100644 .github/workflows/deploy-staging.yml create mode 100644 .github/workflows/pre-commit.yml diff --git a/.github/workflows/deploy-development.yml b/.github/workflows/deploy-development.yml new file mode 100644 index 0000000..1bf5932 --- /dev/null +++ b/.github/workflows/deploy-development.yml @@ -0,0 +1,141 @@ +name: Build, Test and Deploy Development + +on: + pull_request: + branches: + - master + +permissions: + id-token: write # This is required for requesting the JWT + contents: read # This is required for actions/checkout + +jobs: + build_test_push: + runs-on: ubuntu-latest + environment: development + outputs: + image_tag: ${{ steps.set_image_tag.outputs.image_tag }} + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Set up QEMU + uses: docker/setup-qemu-action@v3 + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + + - name: Setup Docker Structure Test + run: > + curl -LO + https://storage.googleapis.com/container-structure-test/latest/container-structure-test-linux-amd64 + && chmod +x container-structure-test-linux-amd64 && sudo mv container-structure-test-linux-amd64 + /usr/local/bin/container-structure-test + + - name: Set Image Tag + id: set_image_tag + run: | + branch_name=${{ github.head_ref || github.ref_name }} + tag=${{ env.TAG_PREFIX}}-${branch_name//\//-} + echo "$tag" + echo "image_tag=$tag" >> $GITHUB_OUTPUT + env: + TAG_PREFIX: dev + + - name: Set up JDK 17 + uses: actions/setup-java@v3 + with: + distribution: 'temurin' + java-version: '17' + cache: 'maven' + + - name: Build with Maven + run: mvn -B package --file pom.xml + + - name: Build Docker Image + uses: docker/build-push-action@v5 + with: + context: . + load: true + tags: ${{ vars.ECR_REPOSITORY }}:${{ steps.set_image_tag.outputs.image_tag }} + + - name: Test Docker Image + run: | + container-structure-test test --image ${{ vars.ECR_REPOSITORY }}:${{ steps.set_image_tag.outputs.image_tag }} --config tests/config.yaml + + - name: Configure AWS Credentials + uses: aws-actions/configure-aws-credentials@v4 + with: + audience: sts.amazonaws.com + aws-region: ${{ vars.AWS_REGION }} + role-to-assume: ${{ secrets.AWS_ROLE_ARN }} + + - name: Login to ECR + uses: docker/login-action@v3 + with: + registry: ${{ vars.ECR_REGISTRY }} + + - name: Build and Push Docker Image + id: build_and_push + uses: docker/build-push-action@v5 + with: + context: . +# Only building for AMD64 for now +# platforms: linux/amd64,linux/arm64 + push: true + tags: ${{ vars.ECR_REGISTRY }}/${{ vars.ECR_REPOSITORY }}:${{ steps.set_image_tag.outputs.image_tag }} + + development_deploy: + runs-on: ubuntu-latest + environment: development + env: + tf_version: '1.5.7' + tg_version: '0.54.0' + tg_dir: './deploy/tg' + needs: [build_test_push] + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Configure AWS Credentials + uses: aws-actions/configure-aws-credentials@v4 + with: + audience: sts.amazonaws.com + aws-region: ${{ vars.AWS_REGION }} + role-to-assume: ${{ secrets.AWS_ROLE_ARN }} + + - name: Expose github environment as shell variables + env: + SECRETS_CONTEXT: ${{ toJson(secrets) }} + VARS_CONTEXT: ${{ toJson(vars) }} + run: | + EOF=$(dd if=/dev/urandom bs=15 count=1 status=none | base64) + to_envs() { jq -r "to_entries[] | \"\(.key)<<$EOF\n\(.value)\n$EOF\n\""; } + echo "$VARS_CONTEXT" | to_envs >> $GITHUB_ENV + echo "$SECRETS_CONTEXT" | to_envs >> $GITHUB_ENV + + - name: Terragrunt Plan + uses: gruntwork-io/terragrunt-action@v2 + with: + tf_version: ${{ env.tf_version }} + tg_version: ${{ env.tg_version }} + tg_dir: ${{ env.tg_dir }} + tg_command: 'run-all plan -out=tf.plan' + env: + TF_INPUT: 0 + TF_IN_AUTOMATION: true + # get the image digest from the build job with optional override from vars context + TF_VAR_image: ${{ vars.IMAGE || needs.build_test_push.outputs.image_tag }} + + - name: Terragrunt Apply + uses: gruntwork-io/terragrunt-action@v2 + with: + tf_version: ${{ env.tf_version }} + tg_version: ${{ env.tg_version }} + tg_dir: ${{ env.tg_dir }} + tg_command: '--terragrunt-non-interactive --terragrunt-log-level info run-all apply -auto-approve tf.plan' + env: + TF_INPUT: 0 + TF_IN_AUTOMATION: true + # get the image digest from the build job with optional override from vars context + TF_VAR_image: ${{ vars.IMAGE || needs.build_test_push.outputs.image_tag }} diff --git a/.github/workflows/deploy-production.yml b/.github/workflows/deploy-production.yml new file mode 100644 index 0000000..d090eea --- /dev/null +++ b/.github/workflows/deploy-production.yml @@ -0,0 +1,132 @@ +name: Deploy Production + +on: + release: + types: + - published + workflow_dispatch: + +permissions: + id-token: write + contents: read + +jobs: + build_push: + runs-on: ubuntu-latest + environment: production + outputs: + image_digest: ${{ steps.build_and_push.outputs.digest || steps.get_digest_from_tagged_image.outputs.image_tag }} + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Set up QEMU + uses: docker/setup-qemu-action@v3 + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + + - name: Set up JDK 17 + uses: actions/setup-java@v3 + with: + distribution: 'temurin' + java-version: '17' + cache: 'maven' + + - name: Build with Maven + run: mvn -B package --file pom.xml + + - name: Configure AWS Credentials + uses: aws-actions/configure-aws-credentials@v4 + with: + audience: sts.amazonaws.com + aws-region: ${{ vars.AWS_REGION }} + role-to-assume: ${{ secrets.AWS_ROLE_ARN }} + + - name: Login to ECR + uses: docker/login-action@v3 + with: + registry: ${{ vars.ECR_REGISTRY }} + + - name: Set Image Tag + id: set_image_tag + run: | + tag=${{ github.ref_name }} + echo "image_tag=${tag//\v/}" >> $GITHUB_OUTPUT + + - name: Build and Push Docker Image + if: ${{ github.event_name != 'workflow_dispatch' }} + id: build_and_push + uses: docker/build-push-action@v5 + with: + context: . +# Only building for AMD64 for now +# platforms: linux/amd64,linux/arm64 + push: true + tags: ${{ vars.ECR_REGISTRY }}/${{ vars.ECR_REPOSITORY }}:${{ steps.set_image_tag.outputs.image_tag }} + + # For manually triggered runs, grab the image digest from the already built image + - name: Get Digest from Tagged Image + if: ${{ github.event_name == 'workflow_dispatch' }} + id: get_digest_from_tagged_image + run: | + image_digest=$(aws ecr describe-images \ + --repository-name ${{ vars.ECR_REPOSITORY }} \ + --image-ids imageTag=${{ steps.set_image_tag.outputs.image_tag }} \ + | jq -r '.imageDetails[].imageDigest') + echo "image_digest=$image_digest" >> $GITHUB_OUTPUT + + production_deploy: + runs-on: ubuntu-latest + environment: production + env: + tf_version: '1.5.7' + tg_version: '0.54.0' + tg_dir: './deploy/tg' + needs: [build_push] + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Configure AWS Credentials + uses: aws-actions/configure-aws-credentials@v4 + with: + audience: sts.amazonaws.com + aws-region: ${{ vars.AWS_REGION }} + role-to-assume: ${{ secrets.AWS_ROLE_ARN }} + + - name: Expose github environment as shell variables + env: + SECRETS_CONTEXT: ${{ toJson(secrets) }} + VARS_CONTEXT: ${{ toJson(vars) }} + run: | + EOF=$(dd if=/dev/urandom bs=15 count=1 status=none | base64) + to_envs() { jq -r "to_entries[] | \"\(.key)<<$EOF\n\(.value)\n$EOF\n\""; } + echo "$VARS_CONTEXT" | to_envs >> $GITHUB_ENV + echo "$SECRETS_CONTEXT" | to_envs >> $GITHUB_ENV + + - name: Terragrunt Plan + uses: gruntwork-io/terragrunt-action@v2 + with: + tf_version: ${{ env.tf_version }} + tg_version: ${{ env.tg_version }} + tg_dir: ${{ env.tg_dir }} + tg_command: 'run-all plan -out=tf.plan' + env: + TF_INPUT: 0 + TF_IN_AUTOMATION: true + # get the image digest from the build job with optional override from vars context + TF_VAR_image: ${{ vars.IMAGE || needs.build_push.outputs.image_digest }} + + - name: Terragrunt Apply + uses: gruntwork-io/terragrunt-action@v2 + with: + tf_version: ${{ env.tf_version }} + tg_version: ${{ env.tg_version }} + tg_dir: ${{ env.tg_dir }} + tg_command: '--terragrunt-non-interactive --terragrunt-log-level info run-all apply -auto-approve tf.plan' + env: + TF_INPUT: 0 + TF_IN_AUTOMATION: true + # get the image digest from the build job with optional override from vars context + TF_VAR_image: ${{ vars.IMAGE || needs.build_push.outputs.image_digest }} diff --git a/.github/workflows/deploy-staging.yml b/.github/workflows/deploy-staging.yml new file mode 100644 index 0000000..d3303b4 --- /dev/null +++ b/.github/workflows/deploy-staging.yml @@ -0,0 +1,114 @@ +name: Deploy Staging + +on: + push: + branches: + - master + workflow_dispatch: + +permissions: + id-token: write + contents: read + +jobs: + build_push: + runs-on: ubuntu-latest + environment: staging + outputs: + image_digest: ${{ steps.build_and_push.outputs.digest }} + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Set up QEMU + uses: docker/setup-qemu-action@v3 + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + + - name: Set up JDK 17 + uses: actions/setup-java@v3 + with: + distribution: 'temurin' + java-version: '17' + cache: 'maven' + + - name: Build with Maven + run: mvn -B package --file pom.xml + + - name: Configure AWS Credentials + uses: aws-actions/configure-aws-credentials@v4 + with: + audience: sts.amazonaws.com + aws-region: ${{ vars.AWS_REGION }} + role-to-assume: ${{ secrets.AWS_ROLE_ARN }} + + - name: Login to ECR + uses: docker/login-action@v3 + with: + registry: ${{ vars.ECR_REGISTRY }} + + - name: Build and Push Docker Image + id: build_and_push + uses: docker/build-push-action@v5 + with: + context: . +# Only building for AMD64 for now +# platforms: linux/amd64,linux/arm64 + push: true + tags: ${{ vars.ECR_REGISTRY }}/${{ vars.ECR_REPOSITORY }}:latest + + staging_deploy: + runs-on: ubuntu-latest + environment: staging + env: + tf_version: '1.5.7' + tg_version: '0.54.0' + tg_dir: './deploy/tg' + needs: build_push + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Configure AWS Credentials + uses: aws-actions/configure-aws-credentials@v4 + with: + audience: sts.amazonaws.com + aws-region: ${{ vars.AWS_REGION }} + role-to-assume: ${{ secrets.AWS_ROLE_ARN }} + + - name: Expose github environment as shell variables + env: + SECRETS_CONTEXT: ${{ toJson(secrets) }} + VARS_CONTEXT: ${{ toJson(vars) }} + run: | + EOF=$(dd if=/dev/urandom bs=15 count=1 status=none | base64) + to_envs() { jq -r "to_entries[] | \"\(.key)<<$EOF\n\(.value)\n$EOF\n\""; } + echo "$VARS_CONTEXT" | to_envs >> $GITHUB_ENV + echo "$SECRETS_CONTEXT" | to_envs >> $GITHUB_ENV + + - name: Terragrunt Plan + uses: gruntwork-io/terragrunt-action@v2 + with: + tf_version: ${{ env.tf_version }} + tg_version: ${{ env.tg_version }} + tg_dir: ${{ env.tg_dir }} + tg_command: 'run-all plan -out=tf.plan' + env: + TF_INPUT: 0 + TF_IN_AUTOMATION: true + # get the image digest from the build job with optional override from vars context + TF_VAR_image: ${{ vars.IMAGE || needs.build_push.outputs.image_digest }} + + - name: Terragrunt Apply + uses: gruntwork-io/terragrunt-action@v2 + with: + tf_version: ${{ env.tf_version }} + tg_version: ${{ env.tg_version }} + tg_dir: ${{ env.tg_dir }} + tg_command: '--terragrunt-non-interactive --terragrunt-log-level info run-all apply -auto-approve tf.plan' + env: + TF_INPUT: 0 + TF_IN_AUTOMATION: true + # get the image digest from the build job with optional override from vars context + TF_VAR_image: ${{ vars.IMAGE || needs.build_push.outputs.image_digest }} diff --git a/.github/workflows/pre-commit.yml b/.github/workflows/pre-commit.yml new file mode 100644 index 0000000..37920ad --- /dev/null +++ b/.github/workflows/pre-commit.yml @@ -0,0 +1,32 @@ +name: Run Pre-commit Checks + +on: + pull_request: + branches: + - master + +permissions: + id-token: write + contents: read + +jobs: + pre_commit: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: + python-version: '3.10' + - uses: hashicorp/setup-terraform@v3 + with: + terraform_version: "1.5.7" + - name: Setup Terragrunt + id: setup_terragrunt + run: | + wget https://github.com/gruntwork-io/terragrunt/releases/download/v${tg_version}/terragrunt_linux_amd64 \ + && mv terragrunt_linux_amd64 terragrunt \ + && chmod +x terragrunt \ + && mv terragrunt /usr/local/bin/terragrunt + env: + tg_version: '0.51.0' + - uses: pre-commit/action@v3.0.0 From 254fb4910bc85a0ee13cf130b991cabf66e43775 Mon Sep 17 00:00:00 2001 From: digorgonzola <29941279+digorgonzola@users.noreply.github.com> Date: Tue, 19 Dec 2023 17:52:39 +1100 Subject: [PATCH 06/51] github env vars for dev: update env name to tf_development (temporarily) --- deploy/github/development.env | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/deploy/github/development.env b/deploy/github/development.env index 51f2eb9..9343bf6 100644 --- a/deploy/github/development.env +++ b/deploy/github/development.env @@ -5,7 +5,7 @@ AWS_ACCOUNT_ID=450356697252 AWS_REGION=ap-southeast-2 ECR_REGISTRY=450356697252.dkr.ecr.ap-southeast-2.amazonaws.com ECR_REPOSITORY=geonetwork4 -ENVIRONMENT=development +ENVIRONMENT=tf_development TF_VAR_app_port=80 TF_VAR_nginx_proxy=false From c5e7c66bb8f8c0c7f9f73a59dee322ad1129afbd Mon Sep 17 00:00:00 2001 From: digorgonzola <29941279+digorgonzola@users.noreply.github.com> Date: Tue, 19 Dec 2023 17:55:09 +1100 Subject: [PATCH 07/51] deploy-development.yml: allow workflow dispatch --- .github/workflows/deploy-development.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/deploy-development.yml b/.github/workflows/deploy-development.yml index 1bf5932..69e72fd 100644 --- a/.github/workflows/deploy-development.yml +++ b/.github/workflows/deploy-development.yml @@ -4,6 +4,7 @@ on: pull_request: branches: - master + workflow_dispatch: permissions: id-token: write # This is required for requesting the JWT From f985f43afaf34b1177477c689e5eb3678133849d Mon Sep 17 00:00:00 2001 From: digorgonzola <29941279+digorgonzola@users.noreply.github.com> Date: Tue, 19 Dec 2023 17:58:06 +1100 Subject: [PATCH 08/51] deploy-development.yml: temporary push event response --- .github/workflows/deploy-development.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/deploy-development.yml b/.github/workflows/deploy-development.yml index 69e72fd..cd42e34 100644 --- a/.github/workflows/deploy-development.yml +++ b/.github/workflows/deploy-development.yml @@ -1,10 +1,10 @@ name: Build, Test and Deploy Development on: - pull_request: - branches: - - master workflow_dispatch: + push: + branches: + - devops_cicd permissions: id-token: write # This is required for requesting the JWT From 76d91e7e9d785b99807d4e5bd55fe83956ad40e8 Mon Sep 17 00:00:00 2001 From: digorgonzola <29941279+digorgonzola@users.noreply.github.com> Date: Tue, 19 Dec 2023 19:26:27 +1100 Subject: [PATCH 09/51] github vars: update app_port to 8000 --- deploy/github/development.env | 4 ++-- deploy/github/production.env | 2 +- deploy/github/staging.env | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/deploy/github/development.env b/deploy/github/development.env index 9343bf6..e8b60e8 100644 --- a/deploy/github/development.env +++ b/deploy/github/development.env @@ -5,8 +5,8 @@ AWS_ACCOUNT_ID=450356697252 AWS_REGION=ap-southeast-2 ECR_REGISTRY=450356697252.dkr.ecr.ap-southeast-2.amazonaws.com ECR_REPOSITORY=geonetwork4 -ENVIRONMENT=tf_development -TF_VAR_app_port=80 +ENVIRONMENT=tf-development +TF_VAR_app_port=8000 TF_VAR_nginx_proxy=false # container definition variables diff --git a/deploy/github/production.env b/deploy/github/production.env index b98434a..1b64926 100644 --- a/deploy/github/production.env +++ b/deploy/github/production.env @@ -7,7 +7,7 @@ AWS_REGION=ap-southeast-2 ECR_REGISTRY=450356697252.dkr.ecr.ap-southeast-2.amazonaws.com ECR_REPOSITORY=geonetwork4 ENVIRONMENT=production -TF_VAR_app_port=80 +TF_VAR_app_port=8000 TF_VAR_nginx_proxy=false # container definition variables diff --git a/deploy/github/staging.env b/deploy/github/staging.env index 8e1b2a3..09fede6 100644 --- a/deploy/github/staging.env +++ b/deploy/github/staging.env @@ -7,7 +7,7 @@ AWS_REGION=ap-southeast-2 ECR_REGISTRY=450356697252.dkr.ecr.ap-southeast-2.amazonaws.com ECR_REPOSITORY=geonetwork4 ENVIRONMENT=staging -TF_VAR_app_port=80 +TF_VAR_app_port=8000 TF_VAR_nginx_proxy=false # container definition variables From 09046de0b008f90886fe9d6336006ddc82e3f42b Mon Sep 17 00:00:00 2001 From: digorgonzola <29941279+digorgonzola@users.noreply.github.com> Date: Tue, 19 Dec 2023 20:06:10 +1100 Subject: [PATCH 10/51] deploy-development.yml: update environment --- .github/workflows/deploy-development.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/deploy-development.yml b/.github/workflows/deploy-development.yml index cd42e34..c60e3ce 100644 --- a/.github/workflows/deploy-development.yml +++ b/.github/workflows/deploy-development.yml @@ -13,7 +13,7 @@ permissions: jobs: build_test_push: runs-on: ubuntu-latest - environment: development + environment: tf-development outputs: image_tag: ${{ steps.set_image_tag.outputs.image_tag }} steps: @@ -88,7 +88,7 @@ jobs: development_deploy: runs-on: ubuntu-latest - environment: development + environment: tf-development env: tf_version: '1.5.7' tg_version: '0.54.0' From dfad7a78ac1cda12a00d2bf3fc76d45f2fe02080 Mon Sep 17 00:00:00 2001 From: digorgonzola <29941279+digorgonzola@users.noreply.github.com> Date: Tue, 19 Dec 2023 20:22:24 +1100 Subject: [PATCH 11/51] add sample docker structure tests --- tests/README-tests.md | 17 +++++++++++++++++ tests/config.yaml | 10 ++++++++++ 2 files changed, 27 insertions(+) create mode 100644 tests/README-tests.md create mode 100644 tests/config.yaml diff --git a/tests/README-tests.md b/tests/README-tests.md new file mode 100644 index 0000000..c850d8e --- /dev/null +++ b/tests/README-tests.md @@ -0,0 +1,17 @@ +# Docker Image Structural Testing +The configuration file `test/config.yaml` defines various "structural" checks to perform on the docker image created by `../Dockerfile`. + +The test require the use of the [container-structure-test](https://github.com/GoogleContainerTools/container-structure-test) from Google. + +Follow their documentation to install. + +## Usage +Generate your docker image using your preferred method e.g. +```shell +docker build -t myimage . +``` + +To run the tests against the image, run the following: +```shell +container-structure-test test --image myimage --config tests/config.yaml +``` diff --git a/tests/config.yaml b/tests/config.yaml new file mode 100644 index 0000000..7981f83 --- /dev/null +++ b/tests/config.yaml @@ -0,0 +1,10 @@ +schemaVersion: 2.0.0 + +commandTests: + - name: "say hello world" + command: "bash" + args: + - -c + - | + echo hello && + echo world From 42b2d0815454fc6d200eefccc08948f6d3da3adf Mon Sep 17 00:00:00 2001 From: digorgonzola <29941279+digorgonzola@users.noreply.github.com> Date: Tue, 19 Dec 2023 21:10:10 +1100 Subject: [PATCH 12/51] deploy/tf: fix container health check allow null --- deploy/tf/service.tf | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/deploy/tf/service.tf b/deploy/tf/service.tf index 6a1b9a0..1b1088c 100644 --- a/deploy/tf/service.tf +++ b/deploy/tf/service.tf @@ -22,9 +22,9 @@ locals { "${var.ecr_registry}/${var.ecr_repository}@${var.image}" : "${var.ecr_registry}/${var.ecr_repository}:${var.image}" ) - health_check = { - command = length(var.app_health_check) > 0 ? split(",", var.app_health_check) : [] - } + health_check = length(var.app_health_check) > 0 ? { + command = split(",", var.app_health_check) + } : {} readonly_root_filesystem = false essential = true memory_reservation = 256 From b03d4677aa69a44c05b2f76f7e8d97de0b8190a7 Mon Sep 17 00:00:00 2001 From: digorgonzola <29941279+digorgonzola@users.noreply.github.com> Date: Tue, 19 Dec 2023 21:29:12 +1100 Subject: [PATCH 13/51] deploy-development.yml: export TF_VAR variables with correct case --- .github/workflows/deploy-development.yml | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/.github/workflows/deploy-development.yml b/.github/workflows/deploy-development.yml index c60e3ce..4613291 100644 --- a/.github/workflows/deploy-development.yml +++ b/.github/workflows/deploy-development.yml @@ -115,6 +115,20 @@ jobs: echo "$VARS_CONTEXT" | to_envs >> $GITHUB_ENV echo "$SECRETS_CONTEXT" | to_envs >> $GITHUB_ENV + - name: Export TF_VAR variables to environment with correct case + env: + vars_json: ${{ toJSON(vars) }} + run: | + tf_vars=$(echo $vars_json | jq -r ' + . | with_entries( .key |= ascii_downcase | select(.key | startswith("tf_var"))) + | to_entries + | map("TF_VAR_\(.key| split("tf_var_")[-1])=\(.value)") |.[]') + + for var in "${tf_vars[@]}"; do + echo "$var" + echo "$var" >> $GITHUB_ENV + done + - name: Terragrunt Plan uses: gruntwork-io/terragrunt-action@v2 with: From 1c4d49000584fed4e4c64a4d266639254c48050e Mon Sep 17 00:00:00 2001 From: digorgonzola <29941279+digorgonzola@users.noreply.github.com> Date: Tue, 19 Dec 2023 22:46:41 +1100 Subject: [PATCH 14/51] tf: update health-check --- deploy/tf/alb.tf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/deploy/tf/alb.tf b/deploy/tf/alb.tf index e88b353..0b1e7a5 100644 --- a/deploy/tf/alb.tf +++ b/deploy/tf/alb.tf @@ -7,7 +7,7 @@ resource "aws_lb_target_group" "app" { health_check { enabled = true - path = "/health" + path = "/" } } From 2020fff8c19dc371a75bee02a2559df5d2ea4896 Mon Sep 17 00:00:00 2001 From: digorgonzola <29941279+digorgonzola@users.noreply.github.com> Date: Tue, 19 Dec 2023 23:22:02 +1100 Subject: [PATCH 15/51] deploy/github: update app port to 8080 --- deploy/github/development.env | 2 +- deploy/github/production.env | 2 +- deploy/github/staging.env | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/deploy/github/development.env b/deploy/github/development.env index e8b60e8..fd32bcb 100644 --- a/deploy/github/development.env +++ b/deploy/github/development.env @@ -6,7 +6,7 @@ AWS_REGION=ap-southeast-2 ECR_REGISTRY=450356697252.dkr.ecr.ap-southeast-2.amazonaws.com ECR_REPOSITORY=geonetwork4 ENVIRONMENT=tf-development -TF_VAR_app_port=8000 +TF_VAR_app_port=8080 TF_VAR_nginx_proxy=false # container definition variables diff --git a/deploy/github/production.env b/deploy/github/production.env index 1b64926..bc0022d 100644 --- a/deploy/github/production.env +++ b/deploy/github/production.env @@ -7,7 +7,7 @@ AWS_REGION=ap-southeast-2 ECR_REGISTRY=450356697252.dkr.ecr.ap-southeast-2.amazonaws.com ECR_REPOSITORY=geonetwork4 ENVIRONMENT=production -TF_VAR_app_port=8000 +TF_VAR_app_port=8080 TF_VAR_nginx_proxy=false # container definition variables diff --git a/deploy/github/staging.env b/deploy/github/staging.env index 09fede6..ed06fa7 100644 --- a/deploy/github/staging.env +++ b/deploy/github/staging.env @@ -7,7 +7,7 @@ AWS_REGION=ap-southeast-2 ECR_REGISTRY=450356697252.dkr.ecr.ap-southeast-2.amazonaws.com ECR_REPOSITORY=geonetwork4 ENVIRONMENT=staging -TF_VAR_app_port=8000 +TF_VAR_app_port=8080 TF_VAR_nginx_proxy=false # container definition variables From 2a443ee9c541f3fc7015d9d53dad7efb21f316fe Mon Sep 17 00:00:00 2001 From: digorgonzola <29941279+digorgonzola@users.noreply.github.com> Date: Tue, 19 Dec 2023 23:50:23 +1100 Subject: [PATCH 16/51] tg: prefer use of tfvars.json for variable overrides --- deploy/tg/ecs/default.auto.tfvars.json | 5 +++++ 1 file changed, 5 insertions(+) create mode 100644 deploy/tg/ecs/default.auto.tfvars.json diff --git a/deploy/tg/ecs/default.auto.tfvars.json b/deploy/tg/ecs/default.auto.tfvars.json new file mode 100644 index 0000000..b3e2e34 --- /dev/null +++ b/deploy/tg/ecs/default.auto.tfvars.json @@ -0,0 +1,5 @@ +{ + "app_port": 8080, + "health_check_path": "/geonetwork/srv/eng/catalog.search#/home", + "nginx_proxy": false +} From 083efb7d2c0f41d602db80e371e2502c93e361da Mon Sep 17 00:00:00 2001 From: digorgonzola <29941279+digorgonzola@users.noreply.github.com> Date: Tue, 19 Dec 2023 23:52:09 +1100 Subject: [PATCH 17/51] tf/service: add health_check_path variable --- deploy/tf/alb.tf | 2 +- deploy/tf/variables.tf | 6 ++++++ 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/deploy/tf/alb.tf b/deploy/tf/alb.tf index 0b1e7a5..b8d3a2e 100644 --- a/deploy/tf/alb.tf +++ b/deploy/tf/alb.tf @@ -7,7 +7,7 @@ resource "aws_lb_target_group" "app" { health_check { enabled = true - path = "/" + path = var.health_check_path } } diff --git a/deploy/tf/variables.tf b/deploy/tf/variables.tf index 614e91f..56406c6 100644 --- a/deploy/tf/variables.tf +++ b/deploy/tf/variables.tf @@ -69,6 +69,12 @@ variable "environment" { type = string } +variable "health_check_path" { + description = "The health check path for the ALB target group." + type = string + default = "/health" +} + variable "iam_statements" { description = "List of IAM statements to attach to the task role" type = any From 38de6e81a4d179036e05c95dcb097bce989b0444 Mon Sep 17 00:00:00 2001 From: digorgonzola <29941279+digorgonzola@users.noreply.github.com> Date: Tue, 19 Dec 2023 23:52:43 +1100 Subject: [PATCH 18/51] github: remove TF_VAR from env file --- deploy/github/development.env | 2 -- deploy/github/production.env | 2 -- deploy/github/staging.env | 2 -- 3 files changed, 6 deletions(-) diff --git a/deploy/github/development.env b/deploy/github/development.env index fd32bcb..7b78f9c 100644 --- a/deploy/github/development.env +++ b/deploy/github/development.env @@ -6,8 +6,6 @@ AWS_REGION=ap-southeast-2 ECR_REGISTRY=450356697252.dkr.ecr.ap-southeast-2.amazonaws.com ECR_REPOSITORY=geonetwork4 ENVIRONMENT=tf-development -TF_VAR_app_port=8080 -TF_VAR_nginx_proxy=false # container definition variables ES_HOST=ec2-3-25-64-248.ap-southeast-2.compute.amazonaws.com diff --git a/deploy/github/production.env b/deploy/github/production.env index bc0022d..cfa59a4 100644 --- a/deploy/github/production.env +++ b/deploy/github/production.env @@ -7,8 +7,6 @@ AWS_REGION=ap-southeast-2 ECR_REGISTRY=450356697252.dkr.ecr.ap-southeast-2.amazonaws.com ECR_REPOSITORY=geonetwork4 ENVIRONMENT=production -TF_VAR_app_port=8080 -TF_VAR_nginx_proxy=false # container definition variables ES_HOST=ec2-3-25-64-248.ap-southeast-2.compute.amazonaws.com diff --git a/deploy/github/staging.env b/deploy/github/staging.env index ed06fa7..a3f3e45 100644 --- a/deploy/github/staging.env +++ b/deploy/github/staging.env @@ -7,8 +7,6 @@ AWS_REGION=ap-southeast-2 ECR_REGISTRY=450356697252.dkr.ecr.ap-southeast-2.amazonaws.com ECR_REPOSITORY=geonetwork4 ENVIRONMENT=staging -TF_VAR_app_port=8080 -TF_VAR_nginx_proxy=false # container definition variables ES_HOST=ec2-3-25-64-248.ap-southeast-2.compute.amazonaws.com From dae69554c5795f08ebf60e364ff63c09715f50a6 Mon Sep 17 00:00:00 2001 From: digorgonzola <29941279+digorgonzola@users.noreply.github.com> Date: Wed, 20 Dec 2023 00:17:56 +1100 Subject: [PATCH 19/51] tf: support advanced health check parameters --- deploy/tf/alb.tf | 7 +++++-- deploy/tf/variables.tf | 31 +++++++++++++++++++++++++------ 2 files changed, 30 insertions(+), 8 deletions(-) diff --git a/deploy/tf/alb.tf b/deploy/tf/alb.tf index b8d3a2e..63daccf 100644 --- a/deploy/tf/alb.tf +++ b/deploy/tf/alb.tf @@ -6,8 +6,11 @@ resource "aws_lb_target_group" "app" { vpc_id = local.vpc_id health_check { - enabled = true - path = var.health_check_path + enabled = true + path = var.health_check_path + healthy_threshold = var.healthy_threshold != null ? var.healthy_threshold : null + interval = var.interval != null ? var.interval : null + unhealthy_threshold = var.unhealthy_threshold != null ? var.unhealthy_threshold : null } } diff --git a/deploy/tf/variables.tf b/deploy/tf/variables.tf index 56406c6..4d07ff5 100644 --- a/deploy/tf/variables.tf +++ b/deploy/tf/variables.tf @@ -69,12 +69,6 @@ variable "environment" { type = string } -variable "health_check_path" { - description = "The health check path for the ALB target group." - type = string - default = "/health" -} - variable "iam_statements" { description = "List of IAM statements to attach to the task role" type = any @@ -104,3 +98,28 @@ variable "proxy_port" { type = number default = 80 } + +# Target group health checks +variable "health_check_path" { + description = "The health check path for the ALB target group." + type = string + default = "/health" +} + +variable "healthy_threshold" { + description = "Number of consecutive health check successes required before considering a target healthy. The range is 2-10." + type = number + default = null +} + +variable "interval" { + description = "The amount of time in seconds between health checks." + type = number + default = null +} + +variable "unhealthy_threshold" { + description = "Number of consecutive health check failures required before considering a target unhealthy. The range is 2-10." + type = number + default = null +} From dd8829210b08577d49fd21627f2c6a79fadce71e Mon Sep 17 00:00:00 2001 From: digorgonzola <29941279+digorgonzola@users.noreply.github.com> Date: Wed, 20 Dec 2023 00:18:27 +1100 Subject: [PATCH 20/51] default.auto.tfvars.json: add further default vars --- deploy/tg/ecs/default.auto.tfvars.json | 3 +++ 1 file changed, 3 insertions(+) diff --git a/deploy/tg/ecs/default.auto.tfvars.json b/deploy/tg/ecs/default.auto.tfvars.json index b3e2e34..52d45fd 100644 --- a/deploy/tg/ecs/default.auto.tfvars.json +++ b/deploy/tg/ecs/default.auto.tfvars.json @@ -1,5 +1,8 @@ { "app_port": 8080, + "cpu": 1024, + "memory": 2048, "health_check_path": "/geonetwork/srv/eng/catalog.search#/home", + "unhealthy_threshold": 10, "nginx_proxy": false } From eefa5455c333174fc64d5ec81f55643bcdcf2d03 Mon Sep 17 00:00:00 2001 From: digorgonzola <29941279+digorgonzola@users.noreply.github.com> Date: Wed, 20 Dec 2023 00:36:52 +1100 Subject: [PATCH 21/51] github: update development.env values --- deploy/github/development.env | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/deploy/github/development.env b/deploy/github/development.env index 7b78f9c..e246f80 100644 --- a/deploy/github/development.env +++ b/deploy/github/development.env @@ -8,8 +8,11 @@ ECR_REPOSITORY=geonetwork4 ENVIRONMENT=tf-development # container definition variables -ES_HOST=ec2-3-25-64-248.ap-southeast-2.compute.amazonaws.com -INDEXER_HOST=ec2-3-25-163-152.ap-southeast-2.compute.amazonaws.com +ES_HOST=mvp-discovery-index.kb.ap-southeast-2.aws.found.io +ES_PORT=9243 +ES_PROTOCOL=https +ES_USERNAME=geonetwork +INDEXER_HOST=http://es-indexer.mvp.aodn.org.au INDEXER_PORT=8081 GEONETWORK_DB_TYPE=postgres GEONETWORK_DB_HOST=stefan-db-rds-primary-evaluation.gamma.aodn.org.au From e5dc0a4bff07b50f5584eb5053e1a858fc08b627 Mon Sep 17 00:00:00 2001 From: digorgonzola <29941279+digorgonzola@users.noreply.github.com> Date: Wed, 20 Dec 2023 00:39:34 +1100 Subject: [PATCH 22/51] default.env: include additional variables --- deploy/container/default.env | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/deploy/container/default.env b/deploy/container/default.env index 07044d6..fac6a35 100644 --- a/deploy/container/default.env +++ b/deploy/container/default.env @@ -1,7 +1,11 @@ -ES_HOST=ec2-3-25-64-248.ap-southeast-2.compute.amazonaws.com -INDEXER_HOST=ec2-3-25-163-152.ap-southeast-2.compute.amazonaws.com -INDEXER_PORT=8081 +ES_HOST= +ES_PASSWORD= +ES_PORT= +ES_PROTOCOL= +ES_USERNAME= INDEXER_APIKEY= +INDEXER_HOST= +INDEXER_PORT=8081 GEONETWORK_DB_TYPE=postgres GEONETWORK_DB_HOST=db GEONETWORK_DB_PORT=5432 From fa66b11e86d1d8831c9c628eadfaa30191debc75 Mon Sep 17 00:00:00 2001 From: digorgonzola <29941279+digorgonzola@users.noreply.github.com> Date: Wed, 20 Dec 2023 00:44:46 +1100 Subject: [PATCH 23/51] default.auto.tfvars.json: increase mem to 3GB --- deploy/tg/ecs/default.auto.tfvars.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/deploy/tg/ecs/default.auto.tfvars.json b/deploy/tg/ecs/default.auto.tfvars.json index 52d45fd..3d7fa47 100644 --- a/deploy/tg/ecs/default.auto.tfvars.json +++ b/deploy/tg/ecs/default.auto.tfvars.json @@ -1,7 +1,7 @@ { "app_port": 8080, "cpu": 1024, - "memory": 2048, + "memory": 3072, "health_check_path": "/geonetwork/srv/eng/catalog.search#/home", "unhealthy_threshold": 10, "nginx_proxy": false From 7676676283877eba6c25b20dd8fad0be1bcecbb5 Mon Sep 17 00:00:00 2001 From: digorgonzola <29941279+digorgonzola@users.noreply.github.com> Date: Wed, 20 Dec 2023 01:16:29 +1100 Subject: [PATCH 24/51] update container vars --- deploy/container/default.env | 4 ++-- deploy/github/development.env | 7 ++----- 2 files changed, 4 insertions(+), 7 deletions(-) diff --git a/deploy/container/default.env b/deploy/container/default.env index fac6a35..0314392 100644 --- a/deploy/container/default.env +++ b/deploy/container/default.env @@ -1,7 +1,7 @@ ES_HOST= ES_PASSWORD= -ES_PORT= -ES_PROTOCOL= +ES_PORT=9200 +ES_PROTOCOL=https ES_USERNAME= INDEXER_APIKEY= INDEXER_HOST= diff --git a/deploy/github/development.env b/deploy/github/development.env index e246f80..7b78f9c 100644 --- a/deploy/github/development.env +++ b/deploy/github/development.env @@ -8,11 +8,8 @@ ECR_REPOSITORY=geonetwork4 ENVIRONMENT=tf-development # container definition variables -ES_HOST=mvp-discovery-index.kb.ap-southeast-2.aws.found.io -ES_PORT=9243 -ES_PROTOCOL=https -ES_USERNAME=geonetwork -INDEXER_HOST=http://es-indexer.mvp.aodn.org.au +ES_HOST=ec2-3-25-64-248.ap-southeast-2.compute.amazonaws.com +INDEXER_HOST=ec2-3-25-163-152.ap-southeast-2.compute.amazonaws.com INDEXER_PORT=8081 GEONETWORK_DB_TYPE=postgres GEONETWORK_DB_HOST=stefan-db-rds-primary-evaluation.gamma.aodn.org.au From 442c93cae6d97b7587fa174ae917bf4ecf3d7a94 Mon Sep 17 00:00:00 2001 From: digorgonzola <29941279+digorgonzola@users.noreply.github.com> Date: Wed, 20 Dec 2023 13:21:51 +1100 Subject: [PATCH 25/51] tf: support passing secrets to container definition --- deploy/tf/service.tf | 2 + deploy/tf/variables.tf | 95 ++++++++++++++++++++++++------------------ 2 files changed, 56 insertions(+), 41 deletions(-) diff --git a/deploy/tf/service.tf b/deploy/tf/service.tf index 1b1088c..c88fb6f 100644 --- a/deploy/tf/service.tf +++ b/deploy/tf/service.tf @@ -43,6 +43,7 @@ locals { sourceVolume = "static" } ] + secrets = var.secrets } } nginx_container_definition = { @@ -70,6 +71,7 @@ locals { sourceVolume = "static" } ] + secrets = [] } } } diff --git a/deploy/tf/variables.tf b/deploy/tf/variables.tf index 4d07ff5..48c9434 100644 --- a/deploy/tf/variables.tf +++ b/deploy/tf/variables.tf @@ -1,8 +1,10 @@ +# ssm variables variable "alb_parameter_name" { description = "The parameter name to derive the ALB details from." type = string } +# container variables variable "app_container_name" { description = "The name of the primary application container" type = string @@ -15,6 +17,58 @@ variable "app_health_check" { default = null } +variable "app_port" { + description = "The port to expose on the application container." + type = number + default = 9000 +} + +variable "cpu" { + description = "The CPU capacity to allocate to the task." + type = number + default = 512 +} + +variable "env_vars" { + description = "Map of key/pair values to pass to the container definition." + type = map(any) +} + + +variable "image" { + description = "The digest/tag of the docker image to pull from ECR" + type = string + default = "latest" +} + +variable "memory" { + description = "The CPU capacity to allocate to the task." + type = number + default = 1024 +} + +variable "nginx_proxy" { + description = "Whether or not to side-load an nginx container in the task definition" + type = bool + default = true +} + +variable "proxy_port" { + description = "The port to expose to the load balancer on the container" + type = number + default = 80 +} + +variable "secrets" { + description = "Map of environment variables and secrets to retrieve values from." + type = list(object({ + name = string + valueFrom = string + })) + default = null +} + +# general variables variable "app_hostnames" { description = "Hostnames to associate with the application" type = list(string) @@ -25,24 +79,12 @@ variable "app_name" { type = string } -variable "app_port" { - description = "The port to expose to the nginx proxy on the application container." - type = number - default = 9000 -} - variable "cluster_arn" { description = "ARN of the existing cluster to deploy the service/tasks to." type = string default = "" } -variable "cpu" { - description = "The CPU capacity to allocate to the task." - type = number - default = 512 -} - variable "create_cluster" { description = "Whether or not to create a separate cluster for this deployment. If false, the name of an existing cluster must be provided." type = bool @@ -59,11 +101,6 @@ variable "ecr_repository" { type = string } -variable "env_vars" { - description = "Map of key/pair values to pass to the container definition." - type = map(any) -} - variable "environment" { description = "Environment name to prepend/append to resource names" type = string @@ -75,30 +112,6 @@ variable "iam_statements" { default = [] } -variable "image" { - description = "The digest/tag of the docker image to pull from ECR" - type = string - default = "latest" -} - -variable "memory" { - description = "The CPU capacity to allocate to the task." - type = number - default = 1024 -} - -variable "nginx_proxy" { - description = "Whether or not to side-load an nginx container in the task definition" - type = bool - default = true -} - -variable "proxy_port" { - description = "The port to expose to the load balancer on the container" - type = number - default = 80 -} - # Target group health checks variable "health_check_path" { description = "The health check path for the ALB target group." From 5c3d377bc96866592643d5d8d24110ffbd6ddd9c Mon Sep 17 00:00:00 2001 From: digorgonzola <29941279+digorgonzola@users.noreply.github.com> Date: Wed, 20 Dec 2023 13:22:47 +1100 Subject: [PATCH 26/51] tg: remove default.auto.tfvars.json --- deploy/tg/ecs/default.auto.tfvars.json | 8 -------- 1 file changed, 8 deletions(-) delete mode 100644 deploy/tg/ecs/default.auto.tfvars.json diff --git a/deploy/tg/ecs/default.auto.tfvars.json b/deploy/tg/ecs/default.auto.tfvars.json deleted file mode 100644 index 3d7fa47..0000000 --- a/deploy/tg/ecs/default.auto.tfvars.json +++ /dev/null @@ -1,8 +0,0 @@ -{ - "app_port": 8080, - "cpu": 1024, - "memory": 3072, - "health_check_path": "/geonetwork/srv/eng/catalog.search#/home", - "unhealthy_threshold": 10, - "nginx_proxy": false -} From cb19b0f3131ed322645ee6e87af01549b7ded792 Mon Sep 17 00:00:00 2001 From: digorgonzola <29941279+digorgonzola@users.noreply.github.com> Date: Wed, 20 Dec 2023 13:23:27 +1100 Subject: [PATCH 27/51] tg: gather override vars + secrets from environment --- deploy/tg/ecs/terragrunt.hcl | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/deploy/tg/ecs/terragrunt.hcl b/deploy/tg/ecs/terragrunt.hcl index 44cfe53..c5babc6 100644 --- a/deploy/tg/ecs/terragrunt.hcl +++ b/deploy/tg/ecs/terragrunt.hcl @@ -8,7 +8,7 @@ include "global" { expose = true } -inputs = { +inputs = merge(local.override_vars, { app_name = get_env("APP_NAME") app_health_check = get_env("APP_HEALTH_CHECK", "") cluster_arn = get_env("CLUSTER_ARN", "") @@ -28,7 +28,8 @@ inputs = { ecr_repository = get_env("ECR_REPOSITORY") iam_statements = local.iam_statements -} + secrets = local.secrets +}) locals { global = include.global.locals @@ -55,6 +56,9 @@ locals { aws_region = local.global.aws_region environment = local.global.environment })), []) + + override_vars = try(yamldecode(file("../../tf_vars/${local.global.environment}/variables.yaml"))) + secrets = try(yamldecode(file("../../tf_vars/${local.global.environment}/secrets.yaml"))) } terraform { From 506d7ef509ee847144a56a4a0f184b746f2d2b2e Mon Sep 17 00:00:00 2001 From: digorgonzola <29941279+digorgonzola@users.noreply.github.com> Date: Wed, 20 Dec 2023 13:23:58 +1100 Subject: [PATCH 28/51] deploy: add in environment tf vars + secrets --- deploy/tf_vars/development/secrets.yaml | 2 ++ deploy/tf_vars/development/variables.yaml | 6 ++++++ 2 files changed, 8 insertions(+) create mode 100644 deploy/tf_vars/development/secrets.yaml create mode 100644 deploy/tf_vars/development/variables.yaml diff --git a/deploy/tf_vars/development/secrets.yaml b/deploy/tf_vars/development/secrets.yaml new file mode 100644 index 0000000..c6841b4 --- /dev/null +++ b/deploy/tf_vars/development/secrets.yaml @@ -0,0 +1,2 @@ +- name: ES_PASSWORD + valueFrom: arn:aws:ssm:ap-southeast-2:450356697252:parameter/apps/geonetwork4/tf-development/es_password diff --git a/deploy/tf_vars/development/variables.yaml b/deploy/tf_vars/development/variables.yaml new file mode 100644 index 0000000..f8b38aa --- /dev/null +++ b/deploy/tf_vars/development/variables.yaml @@ -0,0 +1,6 @@ +app_port: 8080 +cpu: 1024 +memory: 3072 +health_check_path: /geonetwork/srv/eng/catalog.search#/home +unhealthy_threshold: 10 +nginx_proxy: false From b9a35013f3d50a2c464fd09b566bcb742177b0d6 Mon Sep 17 00:00:00 2001 From: digorgonzola <29941279+digorgonzola@users.noreply.github.com> Date: Wed, 20 Dec 2023 13:30:32 +1100 Subject: [PATCH 29/51] deploy: add symlink to development (temporary workaround) --- deploy/tf_vars/tf-development | 1 + 1 file changed, 1 insertion(+) create mode 120000 deploy/tf_vars/tf-development diff --git a/deploy/tf_vars/tf-development b/deploy/tf_vars/tf-development new file mode 120000 index 0000000..baec8fc --- /dev/null +++ b/deploy/tf_vars/tf-development @@ -0,0 +1 @@ +development \ No newline at end of file From 069e76dae67acaaa8299383caed128e033d094b4 Mon Sep 17 00:00:00 2001 From: digorgonzola <29941279+digorgonzola@users.noreply.github.com> Date: Wed, 20 Dec 2023 16:18:28 +1100 Subject: [PATCH 30/51] tf_vars: merge secrets in variables.yaml --- deploy/tf_vars/development/secrets.yaml | 2 -- deploy/tf_vars/development/variables.yaml | 10 ++++++++++ 2 files changed, 10 insertions(+), 2 deletions(-) delete mode 100644 deploy/tf_vars/development/secrets.yaml diff --git a/deploy/tf_vars/development/secrets.yaml b/deploy/tf_vars/development/secrets.yaml deleted file mode 100644 index c6841b4..0000000 --- a/deploy/tf_vars/development/secrets.yaml +++ /dev/null @@ -1,2 +0,0 @@ -- name: ES_PASSWORD - valueFrom: arn:aws:ssm:ap-southeast-2:450356697252:parameter/apps/geonetwork4/tf-development/es_password diff --git a/deploy/tf_vars/development/variables.yaml b/deploy/tf_vars/development/variables.yaml index f8b38aa..646e100 100644 --- a/deploy/tf_vars/development/variables.yaml +++ b/deploy/tf_vars/development/variables.yaml @@ -1,6 +1,16 @@ app_port: 8080 cpu: 1024 memory: 3072 + +container_secrets: + - name: ES_PASSWORD + valueFrom: arn:aws:ssm:ap-southeast-2:450356697252:parameter/apps/geonetwork4/tf-development/es_password + - name: GEONETWORK_DB_PASSWORD + valueFrom: arn:aws:ssm:ap-southeast-2:450356697252:parameter/apps/geonetwork4/tf-development/geonetwork_db_password + - name: INDEXER_APIKEY + valueFrom: arn:aws:ssm:ap-southeast-2:450356697252:parameter/apps/geonetwork4/tf-development/indexer_apikey + health_check_path: /geonetwork/srv/eng/catalog.search#/home unhealthy_threshold: 10 + nginx_proxy: false From 2f3e648bfb911575e260d27ed0274dfe224bd64b Mon Sep 17 00:00:00 2001 From: digorgonzola <29941279+digorgonzola@users.noreply.github.com> Date: Wed, 20 Dec 2023 16:18:42 +1100 Subject: [PATCH 31/51] tf: update var name to container_secrets --- deploy/tf/service.tf | 2 +- deploy/tf/variables.tf | 18 +++++++++--------- 2 files changed, 10 insertions(+), 10 deletions(-) diff --git a/deploy/tf/service.tf b/deploy/tf/service.tf index c88fb6f..db3861a 100644 --- a/deploy/tf/service.tf +++ b/deploy/tf/service.tf @@ -43,7 +43,7 @@ locals { sourceVolume = "static" } ] - secrets = var.secrets + secrets = var.container_secrets } } nginx_container_definition = { diff --git a/deploy/tf/variables.tf b/deploy/tf/variables.tf index 48c9434..04ebd49 100644 --- a/deploy/tf/variables.tf +++ b/deploy/tf/variables.tf @@ -23,6 +23,15 @@ variable "app_port" { default = 9000 } +variable "container_secrets" { + description = "Map of environment variables and secrets to retrieve values from." + type = list(object({ + name = string + valueFrom = string + })) + default = null +} + variable "cpu" { description = "The CPU capacity to allocate to the task." type = number @@ -59,15 +68,6 @@ variable "proxy_port" { default = 80 } -variable "secrets" { - description = "Map of environment variables and secrets to retrieve values from." - type = list(object({ - name = string - valueFrom = string - })) - default = null -} - # general variables variable "app_hostnames" { description = "Hostnames to associate with the application" From d017daac24bb9ef07e88ad2c17612c7f38bc00e2 Mon Sep 17 00:00:00 2001 From: digorgonzola <29941279+digorgonzola@users.noreply.github.com> Date: Wed, 20 Dec 2023 16:19:22 +1100 Subject: [PATCH 32/51] tg: remove secrets (merged into vars) --- deploy/tg/ecs/terragrunt.hcl | 2 -- 1 file changed, 2 deletions(-) diff --git a/deploy/tg/ecs/terragrunt.hcl b/deploy/tg/ecs/terragrunt.hcl index c5babc6..58012fd 100644 --- a/deploy/tg/ecs/terragrunt.hcl +++ b/deploy/tg/ecs/terragrunt.hcl @@ -28,7 +28,6 @@ inputs = merge(local.override_vars, { ecr_repository = get_env("ECR_REPOSITORY") iam_statements = local.iam_statements - secrets = local.secrets }) locals { @@ -58,7 +57,6 @@ locals { })), []) override_vars = try(yamldecode(file("../../tf_vars/${local.global.environment}/variables.yaml"))) - secrets = try(yamldecode(file("../../tf_vars/${local.global.environment}/secrets.yaml"))) } terraform { From 82712afd2720716d9a2cc25cd9a4a34da7422d1e Mon Sep 17 00:00:00 2001 From: digorgonzola <29941279+digorgonzola@users.noreply.github.com> Date: Wed, 20 Dec 2023 19:02:01 +1100 Subject: [PATCH 33/51] major refactor to avoid the github variables mess --- .github/workflows/deploy-development.yml | 24 ++------ deploy/container/development.env | 8 +++ deploy/container/production.env | 8 +++ deploy/container/staging.env | 8 +++ deploy/github/development.env | 18 +----- deploy/github/production.env | 17 +----- deploy/github/staging.env | 17 +----- deploy/iam_statements/development.yaml.tftpl | 14 ----- deploy/iam_statements/production.yaml.tftpl | 0 deploy/iam_statements/staging.yaml.tftpl | 0 deploy/tf/service.tf | 20 +++--- deploy/tf/variables.tf | 20 ++++-- deploy/tf_vars/tf-development | 1 - deploy/tg/ecs/.terraform.lock.hcl | 1 + deploy/tg/ecs/terragrunt.hcl | 61 +++---------------- deploy/tg/global.hcl | 18 +++--- deploy/vars/development/environment.yaml | 4 ++ .../development/variables.yaml | 20 ++++++ 18 files changed, 99 insertions(+), 160 deletions(-) create mode 100644 deploy/container/development.env create mode 100644 deploy/container/production.env create mode 100644 deploy/container/staging.env delete mode 100644 deploy/iam_statements/development.yaml.tftpl delete mode 100644 deploy/iam_statements/production.yaml.tftpl delete mode 100644 deploy/iam_statements/staging.yaml.tftpl delete mode 120000 deploy/tf_vars/tf-development create mode 100644 deploy/vars/development/environment.yaml rename deploy/{tf_vars => vars}/development/variables.yaml (54%) diff --git a/.github/workflows/deploy-development.yml b/.github/workflows/deploy-development.yml index 4613291..882d911 100644 --- a/.github/workflows/deploy-development.yml +++ b/.github/workflows/deploy-development.yml @@ -15,7 +15,7 @@ jobs: runs-on: ubuntu-latest environment: tf-development outputs: - image_tag: ${{ steps.set_image_tag.outputs.image_tag }} + image_digest: ${{ steps.build_and_push.outputs.digest }} steps: - name: Checkout uses: actions/checkout@v4 @@ -105,29 +105,13 @@ jobs: aws-region: ${{ vars.AWS_REGION }} role-to-assume: ${{ secrets.AWS_ROLE_ARN }} - - name: Expose github environment as shell variables + - name: Expose github variables to shell as environment variables env: - SECRETS_CONTEXT: ${{ toJson(secrets) }} VARS_CONTEXT: ${{ toJson(vars) }} run: | EOF=$(dd if=/dev/urandom bs=15 count=1 status=none | base64) to_envs() { jq -r "to_entries[] | \"\(.key)<<$EOF\n\(.value)\n$EOF\n\""; } echo "$VARS_CONTEXT" | to_envs >> $GITHUB_ENV - echo "$SECRETS_CONTEXT" | to_envs >> $GITHUB_ENV - - - name: Export TF_VAR variables to environment with correct case - env: - vars_json: ${{ toJSON(vars) }} - run: | - tf_vars=$(echo $vars_json | jq -r ' - . | with_entries( .key |= ascii_downcase | select(.key | startswith("tf_var"))) - | to_entries - | map("TF_VAR_\(.key| split("tf_var_")[-1])=\(.value)") |.[]') - - for var in "${tf_vars[@]}"; do - echo "$var" - echo "$var" >> $GITHUB_ENV - done - name: Terragrunt Plan uses: gruntwork-io/terragrunt-action@v2 @@ -140,7 +124,7 @@ jobs: TF_INPUT: 0 TF_IN_AUTOMATION: true # get the image digest from the build job with optional override from vars context - TF_VAR_image: ${{ vars.IMAGE || needs.build_test_push.outputs.image_tag }} + TF_VAR_image: ${{ needs.build_test_push.outputs.image_digest }} - name: Terragrunt Apply uses: gruntwork-io/terragrunt-action@v2 @@ -153,4 +137,4 @@ jobs: TF_INPUT: 0 TF_IN_AUTOMATION: true # get the image digest from the build job with optional override from vars context - TF_VAR_image: ${{ vars.IMAGE || needs.build_test_push.outputs.image_tag }} + TF_VAR_image: ${{ needs.build_test_push.outputs.image_digest }} diff --git a/deploy/container/development.env b/deploy/container/development.env new file mode 100644 index 0000000..2f3d038 --- /dev/null +++ b/deploy/container/development.env @@ -0,0 +1,8 @@ +ES_HOST=ec2-3-25-64-248.ap-southeast-2.compute.amazonaws.com +INDEXER_HOST=ec2-3-25-163-152.ap-southeast-2.compute.amazonaws.com +INDEXER_PORT=8081 +GEONETWORK_DB_TYPE=postgres +GEONETWORK_DB_HOST=stefan-db-rds-primary-evaluation.gamma.aodn.org.au +GEONETWORK_DB_PORT=5432 +GEONETWORK_DB_NAME=geonetwork +GEONETWORK_DB_USERNAME=geonetwork diff --git a/deploy/container/production.env b/deploy/container/production.env new file mode 100644 index 0000000..2f3d038 --- /dev/null +++ b/deploy/container/production.env @@ -0,0 +1,8 @@ +ES_HOST=ec2-3-25-64-248.ap-southeast-2.compute.amazonaws.com +INDEXER_HOST=ec2-3-25-163-152.ap-southeast-2.compute.amazonaws.com +INDEXER_PORT=8081 +GEONETWORK_DB_TYPE=postgres +GEONETWORK_DB_HOST=stefan-db-rds-primary-evaluation.gamma.aodn.org.au +GEONETWORK_DB_PORT=5432 +GEONETWORK_DB_NAME=geonetwork +GEONETWORK_DB_USERNAME=geonetwork diff --git a/deploy/container/staging.env b/deploy/container/staging.env new file mode 100644 index 0000000..2f3d038 --- /dev/null +++ b/deploy/container/staging.env @@ -0,0 +1,8 @@ +ES_HOST=ec2-3-25-64-248.ap-southeast-2.compute.amazonaws.com +INDEXER_HOST=ec2-3-25-163-152.ap-southeast-2.compute.amazonaws.com +INDEXER_PORT=8081 +GEONETWORK_DB_TYPE=postgres +GEONETWORK_DB_HOST=stefan-db-rds-primary-evaluation.gamma.aodn.org.au +GEONETWORK_DB_PORT=5432 +GEONETWORK_DB_NAME=geonetwork +GEONETWORK_DB_USERNAME=geonetwork diff --git a/deploy/github/development.env b/deploy/github/development.env index 7b78f9c..cd608b7 100644 --- a/deploy/github/development.env +++ b/deploy/github/development.env @@ -1,18 +1,4 @@ -# general environment variables for Terragrunt -ALB_PARAMETER_NAME=shared-alb-devops-sydney -APP_NAME=geonetwork4 -AWS_ACCOUNT_ID=450356697252 -AWS_REGION=ap-southeast-2 +# variables required for GitHub Actions workflows ECR_REGISTRY=450356697252.dkr.ecr.ap-southeast-2.amazonaws.com ECR_REPOSITORY=geonetwork4 -ENVIRONMENT=tf-development - -# container definition variables -ES_HOST=ec2-3-25-64-248.ap-southeast-2.compute.amazonaws.com -INDEXER_HOST=ec2-3-25-163-152.ap-southeast-2.compute.amazonaws.com -INDEXER_PORT=8081 -GEONETWORK_DB_TYPE=postgres -GEONETWORK_DB_HOST=stefan-db-rds-primary-evaluation.gamma.aodn.org.au -GEONETWORK_DB_PORT=5432 -GEONETWORK_DB_NAME=geonetwork -GEONETWORK_DB_USERNAME=geonetwork +ENVIRONMENT=development diff --git a/deploy/github/production.env b/deploy/github/production.env index cfa59a4..4429a02 100644 --- a/deploy/github/production.env +++ b/deploy/github/production.env @@ -1,19 +1,4 @@ -# general environment variables for Terragrunt -ALB_PARAMETER_NAME=shared-alb-devops-sydney -APP_HEALTH_CHECK="CMD-SHELL,uwsgi-is-ready --stats-socket /tmp/statsock > /dev/null 2>&1 || exit 1" -APP_NAME=geonetwork4 -AWS_ACCOUNT_ID=450356697252 -AWS_REGION=ap-southeast-2 +# variables required for GitHub Actions workflows ECR_REGISTRY=450356697252.dkr.ecr.ap-southeast-2.amazonaws.com ECR_REPOSITORY=geonetwork4 ENVIRONMENT=production - -# container definition variables -ES_HOST=ec2-3-25-64-248.ap-southeast-2.compute.amazonaws.com -INDEXER_HOST=ec2-3-25-163-152.ap-southeast-2.compute.amazonaws.com -INDEXER_PORT=8081 -GEONETWORK_DB_TYPE=postgres -GEONETWORK_DB_HOST=stefan-db-rds-primary-evaluation.gamma.aodn.org.au -GEONETWORK_DB_PORT=5432 -GEONETWORK_DB_NAME=geonetwork -GEONETWORK_DB_USERNAME=geonetwork diff --git a/deploy/github/staging.env b/deploy/github/staging.env index a3f3e45..b80d7e4 100644 --- a/deploy/github/staging.env +++ b/deploy/github/staging.env @@ -1,19 +1,4 @@ -# general environment variables for Terragrunt -ALB_PARAMETER_NAME=shared-alb-devops-sydney -APP_HEALTH_CHECK="CMD-SHELL,uwsgi-is-ready --stats-socket /tmp/statsock > /dev/null 2>&1 || exit 1" -APP_NAME=geonetwork4 -AWS_ACCOUNT_ID=450356697252 -AWS_REGION=ap-southeast-2 +# variables required for GitHub Actions workflows ECR_REGISTRY=450356697252.dkr.ecr.ap-southeast-2.amazonaws.com ECR_REPOSITORY=geonetwork4 ENVIRONMENT=staging - -# container definition variables -ES_HOST=ec2-3-25-64-248.ap-southeast-2.compute.amazonaws.com -INDEXER_HOST=ec2-3-25-163-152.ap-southeast-2.compute.amazonaws.com -INDEXER_PORT=8081 -GEONETWORK_DB_TYPE=postgres -GEONETWORK_DB_HOST=stefan-db-rds-primary-evaluation.gamma.aodn.org.au -GEONETWORK_DB_PORT=5432 -GEONETWORK_DB_NAME=geonetwork -GEONETWORK_DB_USERNAME=geonetwork diff --git a/deploy/iam_statements/development.yaml.tftpl b/deploy/iam_statements/development.yaml.tftpl deleted file mode 100644 index b4075ea..0000000 --- a/deploy/iam_statements/development.yaml.tftpl +++ /dev/null @@ -1,14 +0,0 @@ -#- actions: -# - s3:PutObject -# - s3:GetObjectAcl -# - s3:GetObject -# - s3:ListBucket -# - s3:DeleteObject -# - s3:PutObjectAcl -# resources: -# - arn:aws:s3:::sample-bucket-${environment}-${aws_account} -# - arn:aws:s3:::sample-bucket-${environment}-${aws_account}/* -#- actions: -# - secretsmanager:GetSecretValue -# resources: -# - arn:aws:secretsmanager:${aws_region}:${aws_account}:secret:/rds/* diff --git a/deploy/iam_statements/production.yaml.tftpl b/deploy/iam_statements/production.yaml.tftpl deleted file mode 100644 index e69de29..0000000 diff --git a/deploy/iam_statements/staging.yaml.tftpl b/deploy/iam_statements/staging.yaml.tftpl deleted file mode 100644 index e69de29..0000000 diff --git a/deploy/tf/service.tf b/deploy/tf/service.tf index db3861a..c95d31d 100644 --- a/deploy/tf/service.tf +++ b/deploy/tf/service.tf @@ -1,13 +1,4 @@ locals { - nginx_vars = { - app_host = "127.0.0.1" - app_port = var.app_port - listen_port = var.proxy_port - } - - app_container_vars = [for k, v in var.env_vars : { name = upper(k), value = v }] - nginx_container_vars = [for k, v in local.nginx_vars : { name = upper(k), value = v }] - container_definitions = ( var.nginx_proxy ? merge(local.app_container_definition, local.nginx_container_definition) : @@ -28,7 +19,8 @@ locals { readonly_root_filesystem = false essential = true memory_reservation = 256 - environment = local.app_container_vars + environment = var.env_vars + environment_files = var.environment_files port_mappings = [ { name = var.app_container_name @@ -46,6 +38,7 @@ locals { secrets = var.container_secrets } } + nginx_container_definition = { nginx = { name = "nginx" @@ -56,7 +49,12 @@ locals { readonly_root_filesystem = false essential = true memory_reservation = 256 - environment = local.nginx_container_vars + environment = [ + { name = "APP_HOST", value = "127.0.0.1" }, + { name = "APP_PORT", value = var.app_port }, + { name = "LISTEN_PORT", value = var.proxy_port } + ] + environment_files = [] port_mappings = [ { name = "nginx" diff --git a/deploy/tf/variables.tf b/deploy/tf/variables.tf index 04ebd49..cacdf20 100644 --- a/deploy/tf/variables.tf +++ b/deploy/tf/variables.tf @@ -12,9 +12,9 @@ variable "app_container_name" { } variable "app_health_check" { - description = "The health check commmand to run on the docker container." + description = "The health check command to run on the docker container." type = string - default = null + default = "" } variable "app_port" { @@ -39,10 +39,22 @@ variable "cpu" { } variable "env_vars" { - description = "Map of key/pair values to pass to the container definition." - type = map(any) + description = "List of key/pair values to pass to the container definition." + type = list(object({ + value = string + type = string + })) + default = [] } +variable "environment_files" { + description = "A list of files containing the environment variables to pass to a container" + type = list(object({ + value = string + type = string + })) + default = [] +} variable "image" { description = "The digest/tag of the docker image to pull from ECR" diff --git a/deploy/tf_vars/tf-development b/deploy/tf_vars/tf-development deleted file mode 120000 index baec8fc..0000000 --- a/deploy/tf_vars/tf-development +++ /dev/null @@ -1 +0,0 @@ -development \ No newline at end of file diff --git a/deploy/tg/ecs/.terraform.lock.hcl b/deploy/tg/ecs/.terraform.lock.hcl index fad1fd7..92e9a1a 100644 --- a/deploy/tg/ecs/.terraform.lock.hcl +++ b/deploy/tg/ecs/.terraform.lock.hcl @@ -29,6 +29,7 @@ provider "registry.terraform.io/hashicorp/aws" { provider "registry.terraform.io/hashicorp/null" { version = "3.2.2" hashes = [ + "h1:Gef5VGfobY5uokA5nV/zFvWeMNR2Pmq79DH94QnNZPM=", "h1:IMVAUHKoydFrlPrl9OzasDnw/8ntZFerCC9iXw1rXQY=", "zh:3248aae6a2198f3ec8394218d05bd5e42be59f43a3a7c0b71c66ec0df08b69e7", "zh:32b1aaa1c3013d33c245493f4a65465eab9436b454d250102729321a44c8ab9a", diff --git a/deploy/tg/ecs/terragrunt.hcl b/deploy/tg/ecs/terragrunt.hcl index 58012fd..1094f10 100644 --- a/deploy/tg/ecs/terragrunt.hcl +++ b/deploy/tg/ecs/terragrunt.hcl @@ -1,64 +1,21 @@ -#dependency "s3" { -# config_path = "../s3" -# skip_outputs = true -#} +locals { + environment = try(yamldecode(file("../..//vars/${local.environment_name}/environment.yaml"))) + environment_name = get_env("ENVIRONMENT") + vars = try(yamldecode(file("../..//vars/${local.environment_name}/variables.yaml"))) +} include "global" { - path = "../global.hcl" - expose = true + path = "../global.hcl" } -inputs = merge(local.override_vars, { - app_name = get_env("APP_NAME") - app_health_check = get_env("APP_HEALTH_CHECK", "") - cluster_arn = get_env("CLUSTER_ARN", "") - create_cluster = get_env("CREATE_CLUSTER", true) - environment = local.global.environment - - # fetch the shared infrastructure parameter name - alb_parameter_name = get_env("ALB_PARAMETER_NAME") - - # DNS hostnames to associate with the container - app_hostnames = split(",", get_env("APP_HOSTNAMES", local.default_hostname)) - - # container-specific environment variables - env_vars = local.env_vars +inputs = merge(local.vars, { + app_name = local.environment.app_name + environment = local.environment_name ecr_registry = get_env("ECR_REGISTRY") ecr_repository = get_env("ECR_REPOSITORY") - - iam_statements = local.iam_statements }) -locals { - global = include.global.locals - - # container/task environment variables - default_env_vars = { for tuple in regexall("(.*?)=(.*)", file("../../container/default.env")) : tuple[0] => tuple[1] } - - # get any overrides from the environment (e.g. GitHub deployment variables) - override_env_vars = { - for k, v in local.default_env_vars : - k => can(get_env(k)) ? get_env(k) : v - } - - # remove empty values from the override map - env_vars = { - for k, v in local.override_env_vars : k => v if v != "" - } - - default_hostname = join("-", [get_env("APP_NAME"), local.global.environment]) - - iam_statements = try(yamldecode(templatefile("../..//iam_statements/${local.global.environment}.yaml.tftpl", - { - aws_account = local.global.aws_account - aws_region = local.global.aws_region - environment = local.global.environment - })), []) - - override_vars = try(yamldecode(file("../../tf_vars/${local.global.environment}/variables.yaml"))) -} - terraform { source = "../..//tf" } diff --git a/deploy/tg/global.hcl b/deploy/tg/global.hcl index 19e0292..ee0dab3 100644 --- a/deploy/tg/global.hcl +++ b/deploy/tg/global.hcl @@ -1,10 +1,8 @@ locals { - aws_account = get_env("AWS_ACCOUNT_ID") - aws_region = get_env("AWS_REGION") - environment = get_env("ENVIRONMENT") - project_name = get_env("APP_NAME") - state_bucket = "tfstate-${local.aws_account}-${local.aws_region}" - state_key = "apps/${local.project_name}/${local.environment}/${basename(get_terragrunt_dir())}.tfstate" + environment = try(yamldecode(file("..//vars/${local.environment_name}/environment.yaml"))) + environment_name = get_env("ENVIRONMENT") + state_bucket = "tfstate-${local.environment.aws_account_id}-${local.environment.aws_region}" + state_key = "apps/${local.environment.app_name}/${local.environment_name}/${basename(get_terragrunt_dir())}.tfstate" } generate "providers" { @@ -12,15 +10,15 @@ generate "providers" { if_exists = "overwrite_terragrunt" contents = < Date: Wed, 20 Dec 2023 19:17:04 +1100 Subject: [PATCH 34/51] assign correct permissions to task_exec role for pulling s3 env files --- deploy/tf/aws_details.tf | 2 -- deploy/tf/service.tf | 3 ++- deploy/tf/variables.tf | 20 ++++++++++++++------ deploy/vars/development/variables.yaml | 3 +-- 4 files changed, 17 insertions(+), 11 deletions(-) delete mode 100644 deploy/tf/aws_details.tf diff --git a/deploy/tf/aws_details.tf b/deploy/tf/aws_details.tf deleted file mode 100644 index 038d1e2..0000000 --- a/deploy/tf/aws_details.tf +++ /dev/null @@ -1,2 +0,0 @@ -data "aws_caller_identity" "current" {} -data "aws_region" "current" {} diff --git a/deploy/tf/service.tf b/deploy/tf/service.tf index c95d31d..a661b97 100644 --- a/deploy/tf/service.tf +++ b/deploy/tf/service.tf @@ -148,7 +148,8 @@ module "service" { } } - tasks_iam_role_statements = var.iam_statements + task_exec_iam_statements = var.task_exec_iam_statements + tasks_iam_role_statements = var.tasks_iam_role_statements timeouts = { create = "10m" diff --git a/deploy/tf/variables.tf b/deploy/tf/variables.tf index cacdf20..bc7847d 100644 --- a/deploy/tf/variables.tf +++ b/deploy/tf/variables.tf @@ -4,6 +4,20 @@ variable "alb_parameter_name" { type = string } +# task exec role +variable "task_exec_iam_statements" { + description = "A map of IAM policy [statements](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/iam_policy_document#statement) for custom permission usage" + type = any + default = {} +} + +# tasks role +variable "tasks_iam_role_statements" { + description = "A map of IAM policy [statements](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/iam_policy_document#statement) for custom permission usage" + type = any + default = {} +} + # container variables variable "app_container_name" { description = "The name of the primary application container" @@ -118,12 +132,6 @@ variable "environment" { type = string } -variable "iam_statements" { - description = "List of IAM statements to attach to the task role" - type = any - default = [] -} - # Target group health checks variable "health_check_path" { description = "The health check path for the ALB target group." diff --git a/deploy/vars/development/variables.yaml b/deploy/vars/development/variables.yaml index a90cf67..05cc1c8 100644 --- a/deploy/vars/development/variables.yaml +++ b/deploy/vars/development/variables.yaml @@ -19,7 +19,7 @@ environment_files: - value: arn:aws:s3:::app-config-sydney-450356697252/geonetwork4/development.env type: s3 -iam_statements: +task_exec_iam_statements: - actions: - s3:GetObject resources: @@ -29,7 +29,6 @@ iam_statements: resources: - arn:aws:s3:::app-config-sydney-450356697252 - health_check_path: /geonetwork/srv/eng/catalog.search#/home unhealthy_threshold: 10 From 6d6598b0dddf6d80a740166d67531c0fee57e696 Mon Sep 17 00:00:00 2001 From: digorgonzola <29941279+digorgonzola@users.noreply.github.com> Date: Wed, 20 Dec 2023 20:03:05 +1100 Subject: [PATCH 35/51] update development.env variables --- deploy/container/development.env | 3 +++ 1 file changed, 3 insertions(+) diff --git a/deploy/container/development.env b/deploy/container/development.env index 2f3d038..fc13985 100644 --- a/deploy/container/development.env +++ b/deploy/container/development.env @@ -1,4 +1,7 @@ ES_HOST=ec2-3-25-64-248.ap-southeast-2.compute.amazonaws.com +ES_PORT=9200 +ES_PROTOCOL=https +ES_USERNAME=elastic INDEXER_HOST=ec2-3-25-163-152.ap-southeast-2.compute.amazonaws.com INDEXER_PORT=8081 GEONETWORK_DB_TYPE=postgres From 90bad3a239f99e92a929b3bbe59d7da0d1a60b19 Mon Sep 17 00:00:00 2001 From: digorgonzola <29941279+digorgonzola@users.noreply.github.com> Date: Wed, 20 Dec 2023 20:29:08 +1100 Subject: [PATCH 36/51] github workflows: add step to sync .env files to s3 --- .github/workflows/deploy-development.yml | 7 +++++++ .github/workflows/deploy-production.yml | 11 +++++++++-- .github/workflows/deploy-staging.yml | 11 +++++++++-- 3 files changed, 25 insertions(+), 4 deletions(-) diff --git a/.github/workflows/deploy-development.yml b/.github/workflows/deploy-development.yml index 882d911..07756ad 100644 --- a/.github/workflows/deploy-development.yml +++ b/.github/workflows/deploy-development.yml @@ -105,6 +105,13 @@ jobs: aws-region: ${{ vars.AWS_REGION }} role-to-assume: ${{ secrets.AWS_ROLE_ARN }} + - name: Push container environment file to S3 + run: aws s3 sync . $bucket_path --exclude "*" --include "$include.env" + working-directory: ./deploy/container + env: + bucket_path: ${{ vars.CONFIG_BUCKET_PATH }} + include: ${{ vars.ENVIRONMENT }} + - name: Expose github variables to shell as environment variables env: VARS_CONTEXT: ${{ toJson(vars) }} diff --git a/.github/workflows/deploy-production.yml b/.github/workflows/deploy-production.yml index d090eea..4764c9e 100644 --- a/.github/workflows/deploy-production.yml +++ b/.github/workflows/deploy-production.yml @@ -95,6 +95,13 @@ jobs: aws-region: ${{ vars.AWS_REGION }} role-to-assume: ${{ secrets.AWS_ROLE_ARN }} + - name: Push container environment file to S3 + run: aws s3 sync . $bucket_path --exclude "*" --include "$include.env" + working-directory: ./deploy/container + env: + bucket_path: ${{ vars.CONFIG_BUCKET_PATH }} + include: ${{ vars.ENVIRONMENT }} + - name: Expose github environment as shell variables env: SECRETS_CONTEXT: ${{ toJson(secrets) }} @@ -116,7 +123,7 @@ jobs: TF_INPUT: 0 TF_IN_AUTOMATION: true # get the image digest from the build job with optional override from vars context - TF_VAR_image: ${{ vars.IMAGE || needs.build_push.outputs.image_digest }} + TF_VAR_image: ${{ needs.build_push.outputs.image_digest }} - name: Terragrunt Apply uses: gruntwork-io/terragrunt-action@v2 @@ -129,4 +136,4 @@ jobs: TF_INPUT: 0 TF_IN_AUTOMATION: true # get the image digest from the build job with optional override from vars context - TF_VAR_image: ${{ vars.IMAGE || needs.build_push.outputs.image_digest }} + TF_VAR_image: ${{ needs.build_push.outputs.image_digest }} diff --git a/.github/workflows/deploy-staging.yml b/.github/workflows/deploy-staging.yml index d3303b4..5c77011 100644 --- a/.github/workflows/deploy-staging.yml +++ b/.github/workflows/deploy-staging.yml @@ -77,6 +77,13 @@ jobs: aws-region: ${{ vars.AWS_REGION }} role-to-assume: ${{ secrets.AWS_ROLE_ARN }} + - name: Push container environment file to S3 + run: aws s3 sync . $bucket_path --exclude "*" --include "$include.env" + working-directory: ./deploy/container + env: + bucket_path: ${{ vars.CONFIG_BUCKET_PATH }} + include: ${{ vars.ENVIRONMENT }} + - name: Expose github environment as shell variables env: SECRETS_CONTEXT: ${{ toJson(secrets) }} @@ -98,7 +105,7 @@ jobs: TF_INPUT: 0 TF_IN_AUTOMATION: true # get the image digest from the build job with optional override from vars context - TF_VAR_image: ${{ vars.IMAGE || needs.build_push.outputs.image_digest }} + TF_VAR_image: ${{ needs.build_push.outputs.image_digest }} - name: Terragrunt Apply uses: gruntwork-io/terragrunt-action@v2 @@ -111,4 +118,4 @@ jobs: TF_INPUT: 0 TF_IN_AUTOMATION: true # get the image digest from the build job with optional override from vars context - TF_VAR_image: ${{ vars.IMAGE || needs.build_push.outputs.image_digest }} + TF_VAR_image: ${{ needs.build_push.outputs.image_digest }} From 41392319c027367c11942ba7f5e234481fc892c9 Mon Sep 17 00:00:00 2001 From: digorgonzola <29941279+digorgonzola@users.noreply.github.com> Date: Fri, 29 Dec 2023 23:23:22 +1100 Subject: [PATCH 37/51] remove deployment from app repo --- .github/workflows/deploy-development.yml | 62 +-------- deploy/Dockerfile | 26 ---- deploy/README.md | 60 --------- deploy/container/default.env | 14 -- deploy/container/development.env | 11 -- deploy/container/production.env | 8 -- deploy/container/staging.env | 8 -- deploy/docker-compose.yml | 18 --- deploy/github/development.env | 4 - deploy/github/production.env | 4 - deploy/github/staging.env | 4 - deploy/tf/alb.tf | 44 ------ deploy/tf/cluster.tf | 26 ---- deploy/tf/get-parameters.tf | 62 --------- deploy/tf/main.tf | 0 deploy/tf/outputs.tf | 7 - deploy/tf/service.tf | 162 ----------------------- deploy/tf/variables.tf | 158 ---------------------- deploy/tf/versions.tf | 9 -- deploy/tg/ecs/.terraform.lock.hcl | 47 ------- deploy/tg/ecs/terragrunt.hcl | 21 --- deploy/tg/global.hcl | 45 ------- deploy/vars/development/environment.yaml | 4 - deploy/vars/development/variables.yaml | 35 ----- 24 files changed, 1 insertion(+), 838 deletions(-) delete mode 100644 deploy/Dockerfile delete mode 100644 deploy/README.md delete mode 100644 deploy/container/default.env delete mode 100644 deploy/container/development.env delete mode 100644 deploy/container/production.env delete mode 100644 deploy/container/staging.env delete mode 100644 deploy/docker-compose.yml delete mode 100644 deploy/github/development.env delete mode 100644 deploy/github/production.env delete mode 100644 deploy/github/staging.env delete mode 100644 deploy/tf/alb.tf delete mode 100644 deploy/tf/cluster.tf delete mode 100644 deploy/tf/get-parameters.tf delete mode 100644 deploy/tf/main.tf delete mode 100644 deploy/tf/outputs.tf delete mode 100644 deploy/tf/service.tf delete mode 100644 deploy/tf/variables.tf delete mode 100644 deploy/tf/versions.tf delete mode 100644 deploy/tg/ecs/.terraform.lock.hcl delete mode 100644 deploy/tg/ecs/terragrunt.hcl delete mode 100644 deploy/tg/global.hcl delete mode 100644 deploy/vars/development/environment.yaml delete mode 100644 deploy/vars/development/variables.yaml diff --git a/.github/workflows/deploy-development.yml b/.github/workflows/deploy-development.yml index 07756ad..c7a98cc 100644 --- a/.github/workflows/deploy-development.yml +++ b/.github/workflows/deploy-development.yml @@ -1,4 +1,4 @@ -name: Build, Test and Deploy Development +name: Build, Test and Push - Development on: workflow_dispatch: @@ -85,63 +85,3 @@ jobs: # platforms: linux/amd64,linux/arm64 push: true tags: ${{ vars.ECR_REGISTRY }}/${{ vars.ECR_REPOSITORY }}:${{ steps.set_image_tag.outputs.image_tag }} - - development_deploy: - runs-on: ubuntu-latest - environment: tf-development - env: - tf_version: '1.5.7' - tg_version: '0.54.0' - tg_dir: './deploy/tg' - needs: [build_test_push] - steps: - - name: Checkout - uses: actions/checkout@v4 - - - name: Configure AWS Credentials - uses: aws-actions/configure-aws-credentials@v4 - with: - audience: sts.amazonaws.com - aws-region: ${{ vars.AWS_REGION }} - role-to-assume: ${{ secrets.AWS_ROLE_ARN }} - - - name: Push container environment file to S3 - run: aws s3 sync . $bucket_path --exclude "*" --include "$include.env" - working-directory: ./deploy/container - env: - bucket_path: ${{ vars.CONFIG_BUCKET_PATH }} - include: ${{ vars.ENVIRONMENT }} - - - name: Expose github variables to shell as environment variables - env: - VARS_CONTEXT: ${{ toJson(vars) }} - run: | - EOF=$(dd if=/dev/urandom bs=15 count=1 status=none | base64) - to_envs() { jq -r "to_entries[] | \"\(.key)<<$EOF\n\(.value)\n$EOF\n\""; } - echo "$VARS_CONTEXT" | to_envs >> $GITHUB_ENV - - - name: Terragrunt Plan - uses: gruntwork-io/terragrunt-action@v2 - with: - tf_version: ${{ env.tf_version }} - tg_version: ${{ env.tg_version }} - tg_dir: ${{ env.tg_dir }} - tg_command: 'run-all plan -out=tf.plan' - env: - TF_INPUT: 0 - TF_IN_AUTOMATION: true - # get the image digest from the build job with optional override from vars context - TF_VAR_image: ${{ needs.build_test_push.outputs.image_digest }} - - - name: Terragrunt Apply - uses: gruntwork-io/terragrunt-action@v2 - with: - tf_version: ${{ env.tf_version }} - tg_version: ${{ env.tg_version }} - tg_dir: ${{ env.tg_dir }} - tg_command: '--terragrunt-non-interactive --terragrunt-log-level info run-all apply -auto-approve tf.plan' - env: - TF_INPUT: 0 - TF_IN_AUTOMATION: true - # get the image digest from the build job with optional override from vars context - TF_VAR_image: ${{ needs.build_test_push.outputs.image_digest }} diff --git a/deploy/Dockerfile b/deploy/Dockerfile deleted file mode 100644 index 694ea53..0000000 --- a/deploy/Dockerfile +++ /dev/null @@ -1,26 +0,0 @@ -ARG TF_VERSION=1.5.7 - -FROM hashicorp/terraform:$TF_VERSION - -LABEL maintainer="devops@aodn.org.au" - -ARG TG_VERSION=0.54.0 -ARG USER_ID=1000 -ARG GROUP_ID=1000 -ARG USERNAME=appdeploy - -RUN wget https://github.com/gruntwork-io/terragrunt/releases/download/v${TG_VERSION}/terragrunt_linux_amd64 \ - && mv terragrunt_linux_amd64 terragrunt \ - && chmod +x terragrunt \ - && mv terragrunt /usr/local/bin/terragrunt - -RUN apk add --no-cache aws-cli - -RUN echo 'alias tf="terraform"' >> /etc/profile.d/custom_aliases.sh \ - && echo 'alias tg="terragrunt"' >> /etc/profile.d/custom_aliases.sh - -ENV ENV="/etc/profile" - -RUN adduser -D -u $USER_ID -h /home/$USERNAME $USERNAME - -USER $USERNAME:$USERNAME diff --git a/deploy/README.md b/deploy/README.md deleted file mode 100644 index 53fa71b..0000000 --- a/deploy/README.md +++ /dev/null @@ -1,60 +0,0 @@ -# Deployments -Deployment of this application uses [Github Deployment Environments](https://docs.github.com/en/actions/deployment/targeting-different-environments/using-environments-for-deployment). - -A successful deployment relies on the correct variables being defined in order to deploy to the correct AWS account etc. - -### Managing Environments -You can view the current environment settings by visiting https://github.com/aodn/sample-django-app/settings/environments. - -You can view the currently defined variables there or from the cli using: -```bash -gh variable list -R aodn/sample-django-app -e staging -``` - -### Updating Variables -Manually updating vars can be tedious and error-prone. Instead, you may define the variables you need as a .env file and push these values: -```bash -gh variable set -R aodn/sample-django-app -e staging -f staging.env -``` - -### Deploying From the CLI -The Terragrunt module for this application depends on variables being present and will fail if they are not. - -To test locally, you may want to populate these into your local environment using the following command: -```bash -set -a; source ./github/staging.env; set +a -terragrunt plan -terragrunt apply -``` - -### Using Docker -A Dockerfile is provided to simplify local deployment, removing the need to install the required binaries on the local system. - -N.B. The Dockerfile does assume a valid AWS CLI configuration. - -### Example deployment -Modify the environment variables in `dev.env`: -```text -ALB_PARAMETER_NAME=shared-alb-dev-sydney -APP_NAME=sample-django-app-mybranch -AWS_ACCOUNT_ID=123456789012 -AWS_REGION=ap-southeast-2 -ECR_PARAMETER_NAME=api -ECR_REGISTRY=123456789012.dkr.ecr.ap-southeast-2.amazonaws.com -ECR_REPOSITORY=api -ENVIRONMENT=mydev-stack -RDS_PARAMETER_NAME=db01/primary/development -``` - -```bash -export AWS_PROFILE=myprofile -cd deploy -docker-compose -f docker-compose.yml run terragrunt -``` - -On the container run the following: -```bash -set -a; source ./github/dev.env; set +a -TF_VAR_image=latest terragrunt plan -out=tf.plan -TF_VAR_image=latest terragrunt apply -auto-approve tf.plan -``` diff --git a/deploy/container/default.env b/deploy/container/default.env deleted file mode 100644 index 0314392..0000000 --- a/deploy/container/default.env +++ /dev/null @@ -1,14 +0,0 @@ -ES_HOST= -ES_PASSWORD= -ES_PORT=9200 -ES_PROTOCOL=https -ES_USERNAME= -INDEXER_APIKEY= -INDEXER_HOST= -INDEXER_PORT=8081 -GEONETWORK_DB_TYPE=postgres -GEONETWORK_DB_HOST=db -GEONETWORK_DB_PORT=5432 -GEONETWORK_DB_NAME=geonetwork -GEONETWORK_DB_USERNAME=geonetwork -GEONETWORK_DB_PASSWORD= diff --git a/deploy/container/development.env b/deploy/container/development.env deleted file mode 100644 index fc13985..0000000 --- a/deploy/container/development.env +++ /dev/null @@ -1,11 +0,0 @@ -ES_HOST=ec2-3-25-64-248.ap-southeast-2.compute.amazonaws.com -ES_PORT=9200 -ES_PROTOCOL=https -ES_USERNAME=elastic -INDEXER_HOST=ec2-3-25-163-152.ap-southeast-2.compute.amazonaws.com -INDEXER_PORT=8081 -GEONETWORK_DB_TYPE=postgres -GEONETWORK_DB_HOST=stefan-db-rds-primary-evaluation.gamma.aodn.org.au -GEONETWORK_DB_PORT=5432 -GEONETWORK_DB_NAME=geonetwork -GEONETWORK_DB_USERNAME=geonetwork diff --git a/deploy/container/production.env b/deploy/container/production.env deleted file mode 100644 index 2f3d038..0000000 --- a/deploy/container/production.env +++ /dev/null @@ -1,8 +0,0 @@ -ES_HOST=ec2-3-25-64-248.ap-southeast-2.compute.amazonaws.com -INDEXER_HOST=ec2-3-25-163-152.ap-southeast-2.compute.amazonaws.com -INDEXER_PORT=8081 -GEONETWORK_DB_TYPE=postgres -GEONETWORK_DB_HOST=stefan-db-rds-primary-evaluation.gamma.aodn.org.au -GEONETWORK_DB_PORT=5432 -GEONETWORK_DB_NAME=geonetwork -GEONETWORK_DB_USERNAME=geonetwork diff --git a/deploy/container/staging.env b/deploy/container/staging.env deleted file mode 100644 index 2f3d038..0000000 --- a/deploy/container/staging.env +++ /dev/null @@ -1,8 +0,0 @@ -ES_HOST=ec2-3-25-64-248.ap-southeast-2.compute.amazonaws.com -INDEXER_HOST=ec2-3-25-163-152.ap-southeast-2.compute.amazonaws.com -INDEXER_PORT=8081 -GEONETWORK_DB_TYPE=postgres -GEONETWORK_DB_HOST=stefan-db-rds-primary-evaluation.gamma.aodn.org.au -GEONETWORK_DB_PORT=5432 -GEONETWORK_DB_NAME=geonetwork -GEONETWORK_DB_USERNAME=geonetwork diff --git a/deploy/docker-compose.yml b/deploy/docker-compose.yml deleted file mode 100644 index ebc90ea..0000000 --- a/deploy/docker-compose.yml +++ /dev/null @@ -1,18 +0,0 @@ -version: '3.7' - -services: - terragrunt: - build: - context: . - dockerfile: Dockerfile - args: - TF_VERSION: 1.5.7 - TG_VERSION: 0.54.0 - image: appdeploy:latest - volumes: - - .:/deploy - - ${HOME}/.aws:/home/appdeploy/.aws - working_dir: /deploy/tg - environment: - - AWS_PROFILE=${AWS_PROFILE} - entrypoint: ["sh"] diff --git a/deploy/github/development.env b/deploy/github/development.env deleted file mode 100644 index cd608b7..0000000 --- a/deploy/github/development.env +++ /dev/null @@ -1,4 +0,0 @@ -# variables required for GitHub Actions workflows -ECR_REGISTRY=450356697252.dkr.ecr.ap-southeast-2.amazonaws.com -ECR_REPOSITORY=geonetwork4 -ENVIRONMENT=development diff --git a/deploy/github/production.env b/deploy/github/production.env deleted file mode 100644 index 4429a02..0000000 --- a/deploy/github/production.env +++ /dev/null @@ -1,4 +0,0 @@ -# variables required for GitHub Actions workflows -ECR_REGISTRY=450356697252.dkr.ecr.ap-southeast-2.amazonaws.com -ECR_REPOSITORY=geonetwork4 -ENVIRONMENT=production diff --git a/deploy/github/staging.env b/deploy/github/staging.env deleted file mode 100644 index b80d7e4..0000000 --- a/deploy/github/staging.env +++ /dev/null @@ -1,4 +0,0 @@ -# variables required for GitHub Actions workflows -ECR_REGISTRY=450356697252.dkr.ecr.ap-southeast-2.amazonaws.com -ECR_REPOSITORY=geonetwork4 -ENVIRONMENT=staging diff --git a/deploy/tf/alb.tf b/deploy/tf/alb.tf deleted file mode 100644 index 63daccf..0000000 --- a/deploy/tf/alb.tf +++ /dev/null @@ -1,44 +0,0 @@ -resource "aws_lb_target_group" "app" { - name = "${var.app_name}-${var.environment}" - port = var.nginx_proxy ? var.proxy_port : var.app_port - protocol = "HTTP" - target_type = "ip" - vpc_id = local.vpc_id - - health_check { - enabled = true - path = var.health_check_path - healthy_threshold = var.healthy_threshold != null ? var.healthy_threshold : null - interval = var.interval != null ? var.interval : null - unhealthy_threshold = var.unhealthy_threshold != null ? var.unhealthy_threshold : null - } -} - -resource "aws_route53_record" "app" { - for_each = toset(var.app_hostnames) - zone_id = local.domain_zone_id - name = each.value - type = "A" - - alias { - name = local.alb_dns_name - zone_id = local.alb_zone_id - evaluate_target_health = true - } -} - -resource "aws_lb_listener_rule" "app_fgate" { - for_each = toset(var.app_hostnames) - listener_arn = local.alb_https_listener_arn - - action { - type = "forward" - target_group_arn = aws_lb_target_group.app.arn - } - - condition { - host_header { - values = [aws_route53_record.app[each.value].fqdn] - } - } -} diff --git a/deploy/tf/cluster.tf b/deploy/tf/cluster.tf deleted file mode 100644 index 3a7edaa..0000000 --- a/deploy/tf/cluster.tf +++ /dev/null @@ -1,26 +0,0 @@ -module "cluster" { - source = "terraform-aws-modules/ecs/aws//modules/cluster" - version = "~> 5.7.0" - - create = var.create_cluster ? true : false - - # Cluster Configuration - cluster_name = "${var.app_name}-${var.environment}" - cluster_configuration = { - name = "containerInsights" - value = "enabled" - } - create_task_exec_iam_role = true - fargate_capacity_providers = { - FARGATE = { - default_capacity_provider_strategy = { - weight = 50 - } - } - FARGATE_SPOT = { - default_capacity_provider_strategy = { - weight = 50 - } - } - } -} diff --git a/deploy/tf/get-parameters.tf b/deploy/tf/get-parameters.tf deleted file mode 100644 index d80de7c..0000000 --- a/deploy/tf/get-parameters.tf +++ /dev/null @@ -1,62 +0,0 @@ -locals { - # alb values - alb_dns_name = nonsensitive(data.aws_ssm_parameter.alb_dns_name.value) - alb_https_listener_arn = nonsensitive(data.aws_ssm_parameter.alb_https_listener_arn.value) - alb_zone_id = nonsensitive(data.aws_ssm_parameter.alb_zone_id.value) - - # core values - vpc_id = nonsensitive(data.aws_ssm_parameter.vpc_id.value) - vpc_cidr = nonsensitive(data.aws_ssm_parameter.vpc_cidr.value) - domain_name = nonsensitive(data.aws_ssm_parameter.zonename.value) - domain_zone_id = nonsensitive(data.aws_ssm_parameter.zoneid.value) - public_subnets = split(",", nonsensitive(data.aws_ssm_parameter.public_subnets.value)) - public_subnet_cidrs = nonsensitive(data.aws_ssm_parameter.public_subnet_cidrs.value) - private_subnets = split(",", nonsensitive(data.aws_ssm_parameter.private_subnets.value)) - private_subnet_cidrs = nonsensitive(data.aws_ssm_parameter.private_subnet_cidrs.value) -} - -# alb parameters -data "aws_ssm_parameter" "alb_dns_name" { - name = "/apps/alb/${var.alb_parameter_name}/alb_dns_name" -} - -data "aws_ssm_parameter" "alb_https_listener_arn" { - name = "/apps/alb/${var.alb_parameter_name}/alb_https_listener_arn" -} - -data "aws_ssm_parameter" "alb_zone_id" { - name = "/apps/alb/${var.alb_parameter_name}/alb_zone_id" -} - -# core parameters -data "aws_ssm_parameter" "vpc_id" { - name = "/core/vpc_id" -} - -data "aws_ssm_parameter" "vpc_cidr" { - name = "/core/vpc_cidr" -} - -data "aws_ssm_parameter" "public_subnets" { - name = "/core/subnets_public" -} - -data "aws_ssm_parameter" "public_subnet_cidrs" { - name = "/core/subnets_public_cidr" -} - -data "aws_ssm_parameter" "private_subnets" { - name = "/core/subnets_private" -} - -data "aws_ssm_parameter" "private_subnet_cidrs" { - name = "/core/subnets_private_cidr" -} - -data "aws_ssm_parameter" "zonename" { - name = "/core/zone_domain" -} - -data "aws_ssm_parameter" "zoneid" { - name = "/core/zone_id" -} diff --git a/deploy/tf/main.tf b/deploy/tf/main.tf deleted file mode 100644 index e69de29..0000000 diff --git a/deploy/tf/outputs.tf b/deploy/tf/outputs.tf deleted file mode 100644 index 20f61bb..0000000 --- a/deploy/tf/outputs.tf +++ /dev/null @@ -1,7 +0,0 @@ -output "cluster" { - value = module.cluster -} - -output "service" { - value = module.service -} diff --git a/deploy/tf/service.tf b/deploy/tf/service.tf deleted file mode 100644 index a661b97..0000000 --- a/deploy/tf/service.tf +++ /dev/null @@ -1,162 +0,0 @@ -locals { - container_definitions = ( - var.nginx_proxy ? - merge(local.app_container_definition, local.nginx_container_definition) : - local.app_container_definition - ) - - app_container_definition = { - app = { - name = var.app_container_name - image = ( - startswith(var.image, "sha256") ? - "${var.ecr_registry}/${var.ecr_repository}@${var.image}" : - "${var.ecr_registry}/${var.ecr_repository}:${var.image}" - ) - health_check = length(var.app_health_check) > 0 ? { - command = split(",", var.app_health_check) - } : {} - readonly_root_filesystem = false - essential = true - memory_reservation = 256 - environment = var.env_vars - environment_files = var.environment_files - port_mappings = [ - { - name = var.app_container_name - containerPort = var.app_port - hostPort = var.app_port - } - ] - mount_points = [ - { - readOnly = false - containerPath = "/vol/web" - sourceVolume = "static" - } - ] - secrets = var.container_secrets - } - } - - nginx_container_definition = { - nginx = { - name = "nginx" - image = "${var.ecr_registry}/nginx-proxy:latest" - health_check = { - command = ["CMD-SHELL", "curl -so /dev/null http://localhost/health || exit 1"] - } - readonly_root_filesystem = false - essential = true - memory_reservation = 256 - environment = [ - { name = "APP_HOST", value = "127.0.0.1" }, - { name = "APP_PORT", value = var.app_port }, - { name = "LISTEN_PORT", value = var.proxy_port } - ] - environment_files = [] - port_mappings = [ - { - name = "nginx" - containerPort = var.proxy_port - hostPort = var.proxy_port - } - ] - mount_points = [ - { - readOnly = false - containerPath = "/vol/static" - sourceVolume = "static" - } - ] - secrets = [] - } - } -} - -resource "null_resource" "cluster_arn_precondition_check" { - lifecycle { - precondition { - condition = (var.create_cluster == false && var.cluster_arn != "" || var.create_cluster && var.cluster_arn == "") - error_message = "The cluster ARN must be provided if 'create_cluster' is false. If you mean to have this module create the cluster, set 'create_cluster' to true." - } - } -} - -module "service" { - source = "terraform-aws-modules/ecs/aws//modules/service" - version = "~> 5.7.0" - - name = "${var.app_name}-${var.environment}" - cluster_arn = var.create_cluster ? module.cluster.arn : var.cluster_arn - capacity_provider_strategy = { - env_strategy = { - base = 0 - capacity_provider = var.environment == "production" ? "FARGATE" : "FARGATE_SPOT" - weight = 100 - } - } - - # allow ECS exec commands on containers (e.g. to get a shell session) - enable_execute_command = true - - # resources - cpu = var.cpu - memory = var.memory - - # do not force a new deployment unless the image digest has changed - force_new_deployment = false - - # wait for service to reach steady state - wait_for_steady_state = true - - # wait for the task to reach steady state - wait_until_stable = true - - # Container definition(s) - container_definitions = local.container_definitions - - deployment_circuit_breaker = { - enable = true - rollback = true - } - - load_balancer = { - service = { - target_group_arn = aws_lb_target_group.app.arn - container_name = var.nginx_proxy ? "nginx" : "app" - container_port = var.nginx_proxy ? var.proxy_port : var.app_port - } - } - - subnet_ids = local.private_subnets - - security_group_rules = { - ingress_vpc = { - type = "ingress" - from_port = var.nginx_proxy ? var.proxy_port : var.app_port - to_port = var.nginx_proxy ? var.proxy_port : var.app_port - protocol = "tcp" - cidr_blocks = [local.vpc_cidr] - } - egress_all = { - type = "egress" - from_port = 0 - to_port = 0 - protocol = "-1" - cidr_blocks = ["0.0.0.0/0"] - } - } - - task_exec_iam_statements = var.task_exec_iam_statements - tasks_iam_role_statements = var.tasks_iam_role_statements - - timeouts = { - create = "10m" - update = "10m" - } - - volume = { - static = {} - } -} diff --git a/deploy/tf/variables.tf b/deploy/tf/variables.tf deleted file mode 100644 index bc7847d..0000000 --- a/deploy/tf/variables.tf +++ /dev/null @@ -1,158 +0,0 @@ -# ssm variables -variable "alb_parameter_name" { - description = "The parameter name to derive the ALB details from." - type = string -} - -# task exec role -variable "task_exec_iam_statements" { - description = "A map of IAM policy [statements](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/iam_policy_document#statement) for custom permission usage" - type = any - default = {} -} - -# tasks role -variable "tasks_iam_role_statements" { - description = "A map of IAM policy [statements](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/iam_policy_document#statement) for custom permission usage" - type = any - default = {} -} - -# container variables -variable "app_container_name" { - description = "The name of the primary application container" - type = string - default = "app" -} - -variable "app_health_check" { - description = "The health check command to run on the docker container." - type = string - default = "" -} - -variable "app_port" { - description = "The port to expose on the application container." - type = number - default = 9000 -} - -variable "container_secrets" { - description = "Map of environment variables and secrets to retrieve values from." - type = list(object({ - name = string - valueFrom = string - })) - default = null -} - -variable "cpu" { - description = "The CPU capacity to allocate to the task." - type = number - default = 512 -} - -variable "env_vars" { - description = "List of key/pair values to pass to the container definition." - type = list(object({ - value = string - type = string - })) - default = [] -} - -variable "environment_files" { - description = "A list of files containing the environment variables to pass to a container" - type = list(object({ - value = string - type = string - })) - default = [] -} - -variable "image" { - description = "The digest/tag of the docker image to pull from ECR" - type = string - default = "latest" -} - -variable "memory" { - description = "The CPU capacity to allocate to the task." - type = number - default = 1024 -} - -variable "nginx_proxy" { - description = "Whether or not to side-load an nginx container in the task definition" - type = bool - default = true -} - -variable "proxy_port" { - description = "The port to expose to the load balancer on the container" - type = number - default = 80 -} - -# general variables -variable "app_hostnames" { - description = "Hostnames to associate with the application" - type = list(string) -} - -variable "app_name" { - description = "The name of the application e.g. sample-django-app" - type = string -} - -variable "cluster_arn" { - description = "ARN of the existing cluster to deploy the service/tasks to." - type = string - default = "" -} - -variable "create_cluster" { - description = "Whether or not to create a separate cluster for this deployment. If false, the name of an existing cluster must be provided." - type = bool - default = true -} - -variable "ecr_registry" { - description = "The registry to pull docker images from." - type = string -} - -variable "ecr_repository" { - description = "The repository to pull the image from." - type = string -} - -variable "environment" { - description = "Environment name to prepend/append to resource names" - type = string -} - -# Target group health checks -variable "health_check_path" { - description = "The health check path for the ALB target group." - type = string - default = "/health" -} - -variable "healthy_threshold" { - description = "Number of consecutive health check successes required before considering a target healthy. The range is 2-10." - type = number - default = null -} - -variable "interval" { - description = "The amount of time in seconds between health checks." - type = number - default = null -} - -variable "unhealthy_threshold" { - description = "Number of consecutive health check failures required before considering a target unhealthy. The range is 2-10." - type = number - default = null -} diff --git a/deploy/tf/versions.tf b/deploy/tf/versions.tf deleted file mode 100644 index 91041e5..0000000 --- a/deploy/tf/versions.tf +++ /dev/null @@ -1,9 +0,0 @@ -terraform { - required_version = "~> 1.5.0" - required_providers { - aws = { - source = "hashicorp/aws" - version = "~> 5.12" - } - } -} diff --git a/deploy/tg/ecs/.terraform.lock.hcl b/deploy/tg/ecs/.terraform.lock.hcl deleted file mode 100644 index 92e9a1a..0000000 --- a/deploy/tg/ecs/.terraform.lock.hcl +++ /dev/null @@ -1,47 +0,0 @@ -# This file is maintained automatically by "terraform init". -# Manual edits may be lost in future updates. - -provider "registry.terraform.io/hashicorp/aws" { - version = "5.29.0" - constraints = ">= 4.9.0, >= 4.66.1, ~> 5.12" - hashes = [ - "h1:+4qYlyPaktjZTFP9UbpEaz55jfwWapdibViCPoRFf+s=", - "h1:6Wb/0B+dNeOBP5KXvlHOmZyX7O6CDEfR+MrbxWI3wfo=", - "h1:SyiKAX/D3ZE9My7P03DrRMf65pNnfSDQXPb0g11lCS0=", - "zh:0453c1c64e51cd7050ce46d9280a0195b9073592508077ebf1a1c45f7026f3f5", - "zh:3ee87d1a2870b61fdcc80f3f96b669dbcc8171aadb821bec0e1fa0e6fb9595b6", - "zh:423c0304eba345167cc37dcd300712f24f03fe4de8eecc15edb0d4f88b29ec79", - "zh:6816ce0ed702263297a8e02467bb712c509a9f6e4f132a152a10f1cc19191a81", - "zh:6feb8a0aedabd778216238e72273f5c2ee86d8841acc3fb3dc9d8014a2bbdc51", - "zh:709ccdc8b37f975d422e7955814671548887613931e234e06249da629b0f2f95", - "zh:76c55744020dbdafea25be634f8ac37c1e371f8c397f73bd89bc270d00ee0834", - "zh:7e48d6fc488b9dbe2fd4bebefa1b485d04da38b11a6799f8cba178173b7f8782", - "zh:951d7ef2adbfb96b1d3e9c4780b2ab4375caf9c6b522a2d023c02ff0698d8e2a", - "zh:9b12af85486a96aedd8d7984b0ff811a4b42e3d88dad1a3fb4c0b580d04fa425", - "zh:b0bf5974bc1a7d2ce3f3a9a31a8238ad15ad02211f1e84c54832541ec4bd5d10", - "zh:cc56d4ab9bcbee95f73dbe90f11d4ff7299b835dddf2b30cfda526a2cccd0f9f", - "zh:cfe3a4394f2f7044e03bb63f4fb9c691926607c6784417ac9c0724943da60d09", - "zh:d6f82e13f33f70de8df480287b5a961ced5606f041d1c589f706b112f68db890", - "zh:fb7be5bcff62d0ca9edd4a1bee4d2ed16e9428e3f9eff3ea4d898ecb234505a3", - ] -} - -provider "registry.terraform.io/hashicorp/null" { - version = "3.2.2" - hashes = [ - "h1:Gef5VGfobY5uokA5nV/zFvWeMNR2Pmq79DH94QnNZPM=", - "h1:IMVAUHKoydFrlPrl9OzasDnw/8ntZFerCC9iXw1rXQY=", - "zh:3248aae6a2198f3ec8394218d05bd5e42be59f43a3a7c0b71c66ec0df08b69e7", - "zh:32b1aaa1c3013d33c245493f4a65465eab9436b454d250102729321a44c8ab9a", - "zh:38eff7e470acb48f66380a73a5c7cdd76cc9b9c9ba9a7249c7991488abe22fe3", - "zh:4c2f1faee67af104f5f9e711c4574ff4d298afaa8a420680b0cb55d7bbc65606", - "zh:544b33b757c0b954dbb87db83a5ad921edd61f02f1dc86c6186a5ea86465b546", - "zh:696cf785090e1e8cf1587499516b0494f47413b43cb99877ad97f5d0de3dc539", - "zh:6e301f34757b5d265ae44467d95306d61bef5e41930be1365f5a8dcf80f59452", - "zh:78d5eefdd9e494defcb3c68d282b8f96630502cac21d1ea161f53cfe9bb483b3", - "zh:913a929070c819e59e94bb37a2a253c228f83921136ff4a7aa1a178c7cce5422", - "zh:aa9015926cd152425dbf86d1abdbc74bfe0e1ba3d26b3db35051d7b9ca9f72ae", - "zh:bb04798b016e1e1d49bcc76d62c53b56c88c63d6f2dfe38821afef17c416a0e1", - "zh:c23084e1b23577de22603cff752e59128d83cfecc2e6819edadd8cf7a10af11e", - ] -} diff --git a/deploy/tg/ecs/terragrunt.hcl b/deploy/tg/ecs/terragrunt.hcl deleted file mode 100644 index 1094f10..0000000 --- a/deploy/tg/ecs/terragrunt.hcl +++ /dev/null @@ -1,21 +0,0 @@ -locals { - environment = try(yamldecode(file("../..//vars/${local.environment_name}/environment.yaml"))) - environment_name = get_env("ENVIRONMENT") - vars = try(yamldecode(file("../..//vars/${local.environment_name}/variables.yaml"))) -} - -include "global" { - path = "../global.hcl" -} - -inputs = merge(local.vars, { - app_name = local.environment.app_name - environment = local.environment_name - - ecr_registry = get_env("ECR_REGISTRY") - ecr_repository = get_env("ECR_REPOSITORY") -}) - -terraform { - source = "../..//tf" -} diff --git a/deploy/tg/global.hcl b/deploy/tg/global.hcl deleted file mode 100644 index ee0dab3..0000000 --- a/deploy/tg/global.hcl +++ /dev/null @@ -1,45 +0,0 @@ -locals { - environment = try(yamldecode(file("..//vars/${local.environment_name}/environment.yaml"))) - environment_name = get_env("ENVIRONMENT") - state_bucket = "tfstate-${local.environment.aws_account_id}-${local.environment.aws_region}" - state_key = "apps/${local.environment.app_name}/${local.environment_name}/${basename(get_terragrunt_dir())}.tfstate" -} - -generate "providers" { - path = "providers.tf" - if_exists = "overwrite_terragrunt" - contents = < Date: Mon, 1 Jan 2024 13:27:47 +1100 Subject: [PATCH 38/51] .pre-commit-config.yaml: remove terraform checks --- .pre-commit-config.yaml | 9 --------- 1 file changed, 9 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index f953723..f670082 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -14,12 +14,3 @@ repos: rev: v8.18.0 hooks: - id: gitleaks - - repo: https://github.com/antonbabenko/pre-commit-terraform - rev: v1.83.4 - hooks: - - id: terraform_fmt - - id: terragrunt_fmt - - id: terraform_validate - args: - - --tf-init-args=-backend=false - - id: terragrunt_fmt From ad498f3d5bf20883d64e5b9ea7ecd20e46454dd7 Mon Sep 17 00:00:00 2001 From: digorgonzola <29941279+digorgonzola@users.noreply.github.com> Date: Mon, 1 Jan 2024 13:38:38 +1100 Subject: [PATCH 39/51] update github workflows only build and push docker images then trigger workflows in appdeploy repo --- ...-development.yml => build-development.yml} | 37 +++++- ...oy-production.yml => build-production.yml} | 109 +++++++--------- .github/workflows/build-staging.yml | 108 ++++++++++++++++ .github/workflows/deploy-staging.yml | 121 ------------------ 4 files changed, 190 insertions(+), 185 deletions(-) rename .github/workflows/{deploy-development.yml => build-development.yml} (70%) rename .github/workflows/{deploy-production.yml => build-production.yml} (50%) create mode 100644 .github/workflows/build-staging.yml delete mode 100644 .github/workflows/deploy-staging.yml diff --git a/.github/workflows/deploy-development.yml b/.github/workflows/build-development.yml similarity index 70% rename from .github/workflows/deploy-development.yml rename to .github/workflows/build-development.yml index c7a98cc..93e9c43 100644 --- a/.github/workflows/deploy-development.yml +++ b/.github/workflows/build-development.yml @@ -1,14 +1,13 @@ name: Build, Test and Push - Development on: - workflow_dispatch: push: branches: - devops_cicd permissions: - id-token: write # This is required for requesting the JWT - contents: read # This is required for actions/checkout + id-token: write + contents: read jobs: build_test_push: @@ -85,3 +84,35 @@ jobs: # platforms: linux/amd64,linux/arm64 push: true tags: ${{ vars.ECR_REGISTRY }}/${{ vars.ECR_REPOSITORY }}:${{ steps.set_image_tag.outputs.image_tag }} + + trigger_deploy: + runs-on: ubuntu-latest + needs: [build_test_push] + steps: + - name: Generate App Token + uses: actions/create-github-app-token@v1 + id: app-token + with: + app-id: ${{ vars.DEPLOY_APP_ID }} + private-key: ${{ secrets.DEPLOY_APP_PRIVATE_KEY }} + owner: ${{ github.repository_owner }} + repositories: "appdeploy" + + - name: Trigger Deploy Workflow + uses: actions/github-script@v7 + with: + github-token: ${{ steps.app-token.outputs.token }} + retries: 3 + retry-exempt-status-codes: 204 + script: | + github.rest.actions.createWorkflowDispatch({ + owner: 'aodn', + repo: 'appdeploy', + workflow_id: 'deploy.yml', + ref: 'main', + inputs: { + app_name: 'geonetwork4', + environment: 'development', + image_tag: '${{ needs.build_test_push.outputs.image_digest }}', + } + }) diff --git a/.github/workflows/deploy-production.yml b/.github/workflows/build-production.yml similarity index 50% rename from .github/workflows/deploy-production.yml rename to .github/workflows/build-production.yml index 4764c9e..11c1df2 100644 --- a/.github/workflows/deploy-production.yml +++ b/.github/workflows/build-production.yml @@ -1,17 +1,16 @@ name: Deploy Production on: - release: - types: - - published - workflow_dispatch: + push: + tags: + - prod permissions: id-token: write contents: read jobs: - build_push: + build_test_push: runs-on: ubuntu-latest environment: production outputs: @@ -26,6 +25,13 @@ jobs: - name: Set up Docker Buildx uses: docker/setup-buildx-action@v3 + - name: Setup Docker Structure Test + run: > + curl -LO + https://storage.googleapis.com/container-structure-test/latest/container-structure-test-linux-amd64 + && chmod +x container-structure-test-linux-amd64 && sudo mv container-structure-test-linux-amd64 + /usr/local/bin/container-structure-test + - name: Set up JDK 17 uses: actions/setup-java@v3 with: @@ -36,6 +42,17 @@ jobs: - name: Build with Maven run: mvn -B package --file pom.xml + - name: Build Docker Image + uses: docker/build-push-action@v5 + with: + context: . + load: true + tags: ${{ vars.ECR_REPOSITORY }}:${{ steps.set_image_tag.outputs.image_tag }} + + - name: Test Docker Image + run: | + container-structure-test test --image ${{ vars.ECR_REPOSITORY }}:${{ steps.set_image_tag.outputs.image_tag }} --config tests/config.yaml + - name: Configure AWS Credentials uses: aws-actions/configure-aws-credentials@v4 with: @@ -76,64 +93,34 @@ jobs: | jq -r '.imageDetails[].imageDigest') echo "image_digest=$image_digest" >> $GITHUB_OUTPUT - production_deploy: + trigger_deploy: runs-on: ubuntu-latest - environment: production - env: - tf_version: '1.5.7' - tg_version: '0.54.0' - tg_dir: './deploy/tg' - needs: [build_push] + needs: [build_test_push] steps: - - name: Checkout - uses: actions/checkout@v4 - - - name: Configure AWS Credentials - uses: aws-actions/configure-aws-credentials@v4 + - name: Generate App Token + uses: actions/create-github-app-token@v1 + id: app-token with: - audience: sts.amazonaws.com - aws-region: ${{ vars.AWS_REGION }} - role-to-assume: ${{ secrets.AWS_ROLE_ARN }} - - - name: Push container environment file to S3 - run: aws s3 sync . $bucket_path --exclude "*" --include "$include.env" - working-directory: ./deploy/container - env: - bucket_path: ${{ vars.CONFIG_BUCKET_PATH }} - include: ${{ vars.ENVIRONMENT }} - - - name: Expose github environment as shell variables - env: - SECRETS_CONTEXT: ${{ toJson(secrets) }} - VARS_CONTEXT: ${{ toJson(vars) }} - run: | - EOF=$(dd if=/dev/urandom bs=15 count=1 status=none | base64) - to_envs() { jq -r "to_entries[] | \"\(.key)<<$EOF\n\(.value)\n$EOF\n\""; } - echo "$VARS_CONTEXT" | to_envs >> $GITHUB_ENV - echo "$SECRETS_CONTEXT" | to_envs >> $GITHUB_ENV + app-id: ${{ vars.DEPLOY_APP_ID }} + private-key: ${{ secrets.DEPLOY_APP_PRIVATE_KEY }} + owner: ${{ github.repository_owner }} + repositories: "appdeploy" - - name: Terragrunt Plan - uses: gruntwork-io/terragrunt-action@v2 - with: - tf_version: ${{ env.tf_version }} - tg_version: ${{ env.tg_version }} - tg_dir: ${{ env.tg_dir }} - tg_command: 'run-all plan -out=tf.plan' - env: - TF_INPUT: 0 - TF_IN_AUTOMATION: true - # get the image digest from the build job with optional override from vars context - TF_VAR_image: ${{ needs.build_push.outputs.image_digest }} - - - name: Terragrunt Apply - uses: gruntwork-io/terragrunt-action@v2 + - name: Trigger Deploy Workflow + uses: actions/github-script@v7 with: - tf_version: ${{ env.tf_version }} - tg_version: ${{ env.tg_version }} - tg_dir: ${{ env.tg_dir }} - tg_command: '--terragrunt-non-interactive --terragrunt-log-level info run-all apply -auto-approve tf.plan' - env: - TF_INPUT: 0 - TF_IN_AUTOMATION: true - # get the image digest from the build job with optional override from vars context - TF_VAR_image: ${{ needs.build_push.outputs.image_digest }} + github-token: ${{ steps.app-token.outputs.token }} + retries: 3 + retry-exempt-status-codes: 204 + script: | + github.rest.actions.createWorkflowDispatch({ + owner: 'aodn', + repo: 'appdeploy', + workflow_id: 'deploy.yml', + ref: 'main', + inputs: { + app_name: 'geonetwork4', + environment: 'production', + image_tag: '${{ needs.build_test_push.outputs.image_digest }}', + } + }) diff --git a/.github/workflows/build-staging.yml b/.github/workflows/build-staging.yml new file mode 100644 index 0000000..b12069d --- /dev/null +++ b/.github/workflows/build-staging.yml @@ -0,0 +1,108 @@ +name: Deploy Staging + +on: + push: + branches: + - master + +permissions: + id-token: write + contents: read + +jobs: + build_test_push: + runs-on: ubuntu-latest + environment: staging + outputs: + image_digest: ${{ steps.build_and_push.outputs.digest }} + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Set up QEMU + uses: docker/setup-qemu-action@v3 + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + + - name: Setup Docker Structure Test + run: > + curl -LO + https://storage.googleapis.com/container-structure-test/latest/container-structure-test-linux-amd64 + && chmod +x container-structure-test-linux-amd64 && sudo mv container-structure-test-linux-amd64 + /usr/local/bin/container-structure-test + + - name: Set up JDK 17 + uses: actions/setup-java@v3 + with: + distribution: 'temurin' + java-version: '17' + cache: 'maven' + + - name: Build with Maven + run: mvn -B package --file pom.xml + + - name: Build Docker Image + uses: docker/build-push-action@v5 + with: + context: . + load: true + tags: ${{ vars.ECR_REPOSITORY }}:${{ steps.set_image_tag.outputs.image_tag }} + + - name: Test Docker Image + run: | + container-structure-test test --image ${{ vars.ECR_REPOSITORY }}:${{ steps.set_image_tag.outputs.image_tag }} --config tests/config.yaml + + - name: Configure AWS Credentials + uses: aws-actions/configure-aws-credentials@v4 + with: + audience: sts.amazonaws.com + aws-region: ${{ vars.AWS_REGION }} + role-to-assume: ${{ secrets.AWS_ROLE_ARN }} + + - name: Login to ECR + uses: docker/login-action@v3 + with: + registry: ${{ vars.ECR_REGISTRY }} + + - name: Build and Push Docker Image + id: build_and_push + uses: docker/build-push-action@v5 + with: + context: . +# Only building for AMD64 for now +# platforms: linux/amd64,linux/arm64 + push: true + tags: ${{ vars.ECR_REGISTRY }}/${{ vars.ECR_REPOSITORY }}:latest + + trigger_deploy: + runs-on: ubuntu-latest + needs: [build_test_push] + steps: + - name: Generate App Token + uses: actions/create-github-app-token@v1 + id: app-token + with: + app-id: ${{ vars.DEPLOY_APP_ID }} + private-key: ${{ secrets.DEPLOY_APP_PRIVATE_KEY }} + owner: ${{ github.repository_owner }} + repositories: "appdeploy" + + - name: Trigger Deploy Workflow + uses: actions/github-script@v7 + with: + github-token: ${{ steps.app-token.outputs.token }} + retries: 3 + retry-exempt-status-codes: 204 + script: | + github.rest.actions.createWorkflowDispatch({ + owner: 'aodn', + repo: 'appdeploy', + workflow_id: 'deploy.yml', + ref: 'main', + inputs: { + app_name: 'geonetwork4', + environment: 'staging', + image_tag: '${{ needs.build_test_push.outputs.image_digest }}', + } + }) diff --git a/.github/workflows/deploy-staging.yml b/.github/workflows/deploy-staging.yml deleted file mode 100644 index 5c77011..0000000 --- a/.github/workflows/deploy-staging.yml +++ /dev/null @@ -1,121 +0,0 @@ -name: Deploy Staging - -on: - push: - branches: - - master - workflow_dispatch: - -permissions: - id-token: write - contents: read - -jobs: - build_push: - runs-on: ubuntu-latest - environment: staging - outputs: - image_digest: ${{ steps.build_and_push.outputs.digest }} - steps: - - name: Checkout - uses: actions/checkout@v4 - - - name: Set up QEMU - uses: docker/setup-qemu-action@v3 - - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v3 - - - name: Set up JDK 17 - uses: actions/setup-java@v3 - with: - distribution: 'temurin' - java-version: '17' - cache: 'maven' - - - name: Build with Maven - run: mvn -B package --file pom.xml - - - name: Configure AWS Credentials - uses: aws-actions/configure-aws-credentials@v4 - with: - audience: sts.amazonaws.com - aws-region: ${{ vars.AWS_REGION }} - role-to-assume: ${{ secrets.AWS_ROLE_ARN }} - - - name: Login to ECR - uses: docker/login-action@v3 - with: - registry: ${{ vars.ECR_REGISTRY }} - - - name: Build and Push Docker Image - id: build_and_push - uses: docker/build-push-action@v5 - with: - context: . -# Only building for AMD64 for now -# platforms: linux/amd64,linux/arm64 - push: true - tags: ${{ vars.ECR_REGISTRY }}/${{ vars.ECR_REPOSITORY }}:latest - - staging_deploy: - runs-on: ubuntu-latest - environment: staging - env: - tf_version: '1.5.7' - tg_version: '0.54.0' - tg_dir: './deploy/tg' - needs: build_push - steps: - - name: Checkout - uses: actions/checkout@v4 - - - name: Configure AWS Credentials - uses: aws-actions/configure-aws-credentials@v4 - with: - audience: sts.amazonaws.com - aws-region: ${{ vars.AWS_REGION }} - role-to-assume: ${{ secrets.AWS_ROLE_ARN }} - - - name: Push container environment file to S3 - run: aws s3 sync . $bucket_path --exclude "*" --include "$include.env" - working-directory: ./deploy/container - env: - bucket_path: ${{ vars.CONFIG_BUCKET_PATH }} - include: ${{ vars.ENVIRONMENT }} - - - name: Expose github environment as shell variables - env: - SECRETS_CONTEXT: ${{ toJson(secrets) }} - VARS_CONTEXT: ${{ toJson(vars) }} - run: | - EOF=$(dd if=/dev/urandom bs=15 count=1 status=none | base64) - to_envs() { jq -r "to_entries[] | \"\(.key)<<$EOF\n\(.value)\n$EOF\n\""; } - echo "$VARS_CONTEXT" | to_envs >> $GITHUB_ENV - echo "$SECRETS_CONTEXT" | to_envs >> $GITHUB_ENV - - - name: Terragrunt Plan - uses: gruntwork-io/terragrunt-action@v2 - with: - tf_version: ${{ env.tf_version }} - tg_version: ${{ env.tg_version }} - tg_dir: ${{ env.tg_dir }} - tg_command: 'run-all plan -out=tf.plan' - env: - TF_INPUT: 0 - TF_IN_AUTOMATION: true - # get the image digest from the build job with optional override from vars context - TF_VAR_image: ${{ needs.build_push.outputs.image_digest }} - - - name: Terragrunt Apply - uses: gruntwork-io/terragrunt-action@v2 - with: - tf_version: ${{ env.tf_version }} - tg_version: ${{ env.tg_version }} - tg_dir: ${{ env.tg_dir }} - tg_command: '--terragrunt-non-interactive --terragrunt-log-level info run-all apply -auto-approve tf.plan' - env: - TF_INPUT: 0 - TF_IN_AUTOMATION: true - # get the image digest from the build job with optional override from vars context - TF_VAR_image: ${{ needs.build_push.outputs.image_digest }} From 06484592fc837ac145965806f356abd64ac544e8 Mon Sep 17 00:00:00 2001 From: Stefan Hattrell <29941279+digorgonzola@users.noreply.github.com> Date: Wed, 24 Jan 2024 12:16:40 +1100 Subject: [PATCH 40/51] update pre-commit.yml: remove terragrunt/terraform --- .github/workflows/pre-commit.yml | 12 ------------ 1 file changed, 12 deletions(-) diff --git a/.github/workflows/pre-commit.yml b/.github/workflows/pre-commit.yml index 37920ad..64730fc 100644 --- a/.github/workflows/pre-commit.yml +++ b/.github/workflows/pre-commit.yml @@ -17,16 +17,4 @@ jobs: - uses: actions/setup-python@v5 with: python-version: '3.10' - - uses: hashicorp/setup-terraform@v3 - with: - terraform_version: "1.5.7" - - name: Setup Terragrunt - id: setup_terragrunt - run: | - wget https://github.com/gruntwork-io/terragrunt/releases/download/v${tg_version}/terragrunt_linux_amd64 \ - && mv terragrunt_linux_amd64 terragrunt \ - && chmod +x terragrunt \ - && mv terragrunt /usr/local/bin/terragrunt - env: - tg_version: '0.51.0' - uses: pre-commit/action@v3.0.0 From f38aeb571aa1096f66fc91d8d3e92f65f92ce012 Mon Sep 17 00:00:00 2001 From: Stefan Hattrell <29941279+digorgonzola@users.noreply.github.com> Date: Wed, 24 Jan 2024 12:17:10 +1100 Subject: [PATCH 41/51] github: add separate test workflow run on pull requests --- .github/workflows/test.yml | 66 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 66 insertions(+) create mode 100644 .github/workflows/test.yml diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml new file mode 100644 index 0000000..5772e65 --- /dev/null +++ b/.github/workflows/test.yml @@ -0,0 +1,66 @@ +name: Test + +on: + pull_request: + branches: + - master + - main + +concurrency: + group: ${{ github.ref }} + cancel-in-progress: true + +permissions: + id-token: write + contents: read + +jobs: + build_test_push: + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Set up QEMU + uses: docker/setup-qemu-action@v3 + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + + - name: Setup Docker Structure Test + run: > + curl -LO + https://storage.googleapis.com/container-structure-test/latest/container-structure-test-linux-amd64 + && chmod +x container-structure-test-linux-amd64 && sudo mv container-structure-test-linux-amd64 + /usr/local/bin/container-structure-test + + - name: Set Image Tag + id: set_image_tag + run: | + branch_name=${{ github.head_ref || github.ref_name }} + tag=${{ env.TAG_PREFIX}}-${branch_name//\//-} + echo "$tag" + echo "image_tag=$tag" >> $GITHUB_OUTPUT + env: + TAG_PREFIX: test + + - name: Set up JDK 17 + uses: actions/setup-java@v3 + with: + distribution: 'temurin' + java-version: '17' + cache: 'maven' + + - name: Build with Maven + run: mvn -B package --file pom.xml + + - name: Build Docker Image + uses: docker/build-push-action@v5 + with: + context: . + load: true + tags: ${{ vars.ECR_REPOSITORY }}:${{ steps.set_image_tag.outputs.image_tag }} + + - name: Test Docker Image + run: | + container-structure-test test --image ${{ vars.ECR_REPOSITORY }}:${{ steps.set_image_tag.outputs.image_tag }} --config tests/config.yaml From b7dd4d038784a9c8c22fd476d4bb239534c7bcb2 Mon Sep 17 00:00:00 2001 From: Stefan Hattrell <29941279+digorgonzola@users.noreply.github.com> Date: Wed, 24 Jan 2024 12:30:24 +1100 Subject: [PATCH 42/51] github: refactor build jobs push to ecr and update SSM parameter with digest to be used by appdeploy repo --- .github/workflows/build-development.yml | 118 ----------------------- .github/workflows/build-production.yml | 123 +++++++++++------------- .github/workflows/build-staging.yml | 109 ++++++++++++--------- 3 files changed, 118 insertions(+), 232 deletions(-) delete mode 100644 .github/workflows/build-development.yml diff --git a/.github/workflows/build-development.yml b/.github/workflows/build-development.yml deleted file mode 100644 index 93e9c43..0000000 --- a/.github/workflows/build-development.yml +++ /dev/null @@ -1,118 +0,0 @@ -name: Build, Test and Push - Development - -on: - push: - branches: - - devops_cicd - -permissions: - id-token: write - contents: read - -jobs: - build_test_push: - runs-on: ubuntu-latest - environment: tf-development - outputs: - image_digest: ${{ steps.build_and_push.outputs.digest }} - steps: - - name: Checkout - uses: actions/checkout@v4 - - - name: Set up QEMU - uses: docker/setup-qemu-action@v3 - - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v3 - - - name: Setup Docker Structure Test - run: > - curl -LO - https://storage.googleapis.com/container-structure-test/latest/container-structure-test-linux-amd64 - && chmod +x container-structure-test-linux-amd64 && sudo mv container-structure-test-linux-amd64 - /usr/local/bin/container-structure-test - - - name: Set Image Tag - id: set_image_tag - run: | - branch_name=${{ github.head_ref || github.ref_name }} - tag=${{ env.TAG_PREFIX}}-${branch_name//\//-} - echo "$tag" - echo "image_tag=$tag" >> $GITHUB_OUTPUT - env: - TAG_PREFIX: dev - - - name: Set up JDK 17 - uses: actions/setup-java@v3 - with: - distribution: 'temurin' - java-version: '17' - cache: 'maven' - - - name: Build with Maven - run: mvn -B package --file pom.xml - - - name: Build Docker Image - uses: docker/build-push-action@v5 - with: - context: . - load: true - tags: ${{ vars.ECR_REPOSITORY }}:${{ steps.set_image_tag.outputs.image_tag }} - - - name: Test Docker Image - run: | - container-structure-test test --image ${{ vars.ECR_REPOSITORY }}:${{ steps.set_image_tag.outputs.image_tag }} --config tests/config.yaml - - - name: Configure AWS Credentials - uses: aws-actions/configure-aws-credentials@v4 - with: - audience: sts.amazonaws.com - aws-region: ${{ vars.AWS_REGION }} - role-to-assume: ${{ secrets.AWS_ROLE_ARN }} - - - name: Login to ECR - uses: docker/login-action@v3 - with: - registry: ${{ vars.ECR_REGISTRY }} - - - name: Build and Push Docker Image - id: build_and_push - uses: docker/build-push-action@v5 - with: - context: . -# Only building for AMD64 for now -# platforms: linux/amd64,linux/arm64 - push: true - tags: ${{ vars.ECR_REGISTRY }}/${{ vars.ECR_REPOSITORY }}:${{ steps.set_image_tag.outputs.image_tag }} - - trigger_deploy: - runs-on: ubuntu-latest - needs: [build_test_push] - steps: - - name: Generate App Token - uses: actions/create-github-app-token@v1 - id: app-token - with: - app-id: ${{ vars.DEPLOY_APP_ID }} - private-key: ${{ secrets.DEPLOY_APP_PRIVATE_KEY }} - owner: ${{ github.repository_owner }} - repositories: "appdeploy" - - - name: Trigger Deploy Workflow - uses: actions/github-script@v7 - with: - github-token: ${{ steps.app-token.outputs.token }} - retries: 3 - retry-exempt-status-codes: 204 - script: | - github.rest.actions.createWorkflowDispatch({ - owner: 'aodn', - repo: 'appdeploy', - workflow_id: 'deploy.yml', - ref: 'main', - inputs: { - app_name: 'geonetwork4', - environment: 'development', - image_tag: '${{ needs.build_test_push.outputs.image_digest }}', - } - }) diff --git a/.github/workflows/build-production.yml b/.github/workflows/build-production.yml index 11c1df2..d0c0998 100644 --- a/.github/workflows/build-production.yml +++ b/.github/workflows/build-production.yml @@ -1,20 +1,20 @@ -name: Deploy Production +name: Build Production on: push: tags: - - prod + - v*.*.* permissions: id-token: write contents: read jobs: - build_test_push: + build_push: runs-on: ubuntu-latest environment: production outputs: - image_digest: ${{ steps.build_and_push.outputs.digest || steps.get_digest_from_tagged_image.outputs.image_tag }} + image_digest: ${{ steps.build_and_push.outputs.digest }} steps: - name: Checkout uses: actions/checkout@v4 @@ -25,13 +25,6 @@ jobs: - name: Set up Docker Buildx uses: docker/setup-buildx-action@v3 - - name: Setup Docker Structure Test - run: > - curl -LO - https://storage.googleapis.com/container-structure-test/latest/container-structure-test-linux-amd64 - && chmod +x container-structure-test-linux-amd64 && sudo mv container-structure-test-linux-amd64 - /usr/local/bin/container-structure-test - - name: Set up JDK 17 uses: actions/setup-java@v3 with: @@ -47,11 +40,7 @@ jobs: with: context: . load: true - tags: ${{ vars.ECR_REPOSITORY }}:${{ steps.set_image_tag.outputs.image_tag }} - - - name: Test Docker Image - run: | - container-structure-test test --image ${{ vars.ECR_REPOSITORY }}:${{ steps.set_image_tag.outputs.image_tag }} --config tests/config.yaml + tags: ${{ vars.ECR_REPOSITORY }}:${{ github.ref_name }} - name: Configure AWS Credentials uses: aws-actions/configure-aws-credentials@v4 @@ -65,62 +54,62 @@ jobs: with: registry: ${{ vars.ECR_REGISTRY }} - - name: Set Image Tag - id: set_image_tag - run: | - tag=${{ github.ref_name }} - echo "image_tag=${tag//\v/}" >> $GITHUB_OUTPUT - - name: Build and Push Docker Image - if: ${{ github.event_name != 'workflow_dispatch' }} id: build_and_push uses: docker/build-push-action@v5 with: context: . -# Only building for AMD64 for now -# platforms: linux/amd64,linux/arm64 + # Only building for AMD64 for now + # platforms: linux/amd64,linux/arm64 push: true - tags: ${{ vars.ECR_REGISTRY }}/${{ vars.ECR_REPOSITORY }}:${{ steps.set_image_tag.outputs.image_tag }} + tags: ${{ vars.ECR_REGISTRY }}/${{ vars.ECR_REPOSITORY }}:${{ github.ref_name }} - # For manually triggered runs, grab the image digest from the already built image - - name: Get Digest from Tagged Image - if: ${{ github.event_name == 'workflow_dispatch' }} - id: get_digest_from_tagged_image + - name: Push Image Digest to SSM run: | - image_digest=$(aws ecr describe-images \ - --repository-name ${{ vars.ECR_REPOSITORY }} \ - --image-ids imageTag=${{ steps.set_image_tag.outputs.image_tag }} \ - | jq -r '.imageDetails[].imageDigest') - echo "image_digest=$image_digest" >> $GITHUB_OUTPUT - - trigger_deploy: - runs-on: ubuntu-latest - needs: [build_test_push] - steps: - - name: Generate App Token - uses: actions/create-github-app-token@v1 - id: app-token - with: - app-id: ${{ vars.DEPLOY_APP_ID }} - private-key: ${{ secrets.DEPLOY_APP_PRIVATE_KEY }} - owner: ${{ github.repository_owner }} - repositories: "appdeploy" - - - name: Trigger Deploy Workflow - uses: actions/github-script@v7 - with: - github-token: ${{ steps.app-token.outputs.token }} - retries: 3 - retry-exempt-status-codes: 204 - script: | - github.rest.actions.createWorkflowDispatch({ - owner: 'aodn', - repo: 'appdeploy', - workflow_id: 'deploy.yml', - ref: 'main', - inputs: { - app_name: 'geonetwork4', - environment: 'production', - image_tag: '${{ needs.build_test_push.outputs.image_digest }}', - } - }) + aws ssm put-parameter \ + --name "/apps/sample-django-app/production/image_digest" \ + --type "String" \ + --value "$digest" \ + --overwrite + env: + digest: ${{ steps.build_and_push.outputs.digest }} + +# Optional deployment job if you want to update the task immediately +# However, the appdeploy repository is configured to run drift jobs on an hourly basis +# See: https://terrateam.io/docs/features/drift-detection + +# deploy: +# runs-on: ubuntu-latest +# environment: production +# needs: [build_push] +# steps: +# - name: Configure AWS Credentials +# uses: aws-actions/configure-aws-credentials@v4 +# with: +# audience: sts.amazonaws.com +# aws-region: ${{ vars.AWS_REGION }} +# role-to-assume: ${{ secrets.AWS_ROLE_ARN }} +# +# - name: Get Currently Running Task Definition +# id: get-current-task-definition +# run: | +# aws ecs describe-task-definition \ +# --task-definition ${{ vars.FAMILY }} \ +# --query taskDefinition > task-definition.json +# +# - name: Update Task Definition with Image Digest +# id: update-api-image-tag +# uses: aws-actions/amazon-ecs-render-task-definition@v1 +# with: +# task-definition: task-definition.json +# container-name: app +# image: ${{ vars.ECR_REGISTRY }}/${{ vars.ECR_REPOSITORY }}@${{ needs.build_push.outputs.image_digest }} +# +# - name: Deploy to Amazon ECS service +# uses: aws-actions/amazon-ecs-deploy-task-definition@v1 +# with: +# task-definition: ${{ steps.update-api-image-tag.outputs.task-definition }} +# service: ${{ vars.FAMILY }} +# cluster: ${{ vars.CLUSTER }} +# force-new-deployment: true +# wait-for-service-stability: true diff --git a/.github/workflows/build-staging.yml b/.github/workflows/build-staging.yml index b12069d..d3d27a9 100644 --- a/.github/workflows/build-staging.yml +++ b/.github/workflows/build-staging.yml @@ -1,16 +1,23 @@ -name: Deploy Staging +name: Build Staging on: push: branches: - master + - main + # This is only here for testing purposes while in a PR + #TODO remove before merge to main + pull_request: + branches: + - master + - main permissions: id-token: write contents: read jobs: - build_test_push: + build_push: runs-on: ubuntu-latest environment: staging outputs: @@ -25,13 +32,6 @@ jobs: - name: Set up Docker Buildx uses: docker/setup-buildx-action@v3 - - name: Setup Docker Structure Test - run: > - curl -LO - https://storage.googleapis.com/container-structure-test/latest/container-structure-test-linux-amd64 - && chmod +x container-structure-test-linux-amd64 && sudo mv container-structure-test-linux-amd64 - /usr/local/bin/container-structure-test - - name: Set up JDK 17 uses: actions/setup-java@v3 with: @@ -47,11 +47,7 @@ jobs: with: context: . load: true - tags: ${{ vars.ECR_REPOSITORY }}:${{ steps.set_image_tag.outputs.image_tag }} - - - name: Test Docker Image - run: | - container-structure-test test --image ${{ vars.ECR_REPOSITORY }}:${{ steps.set_image_tag.outputs.image_tag }} --config tests/config.yaml + tags: ${{ vars.ECR_REPOSITORY }}:${{ github.sha }} - name: Configure AWS Credentials uses: aws-actions/configure-aws-credentials@v4 @@ -70,39 +66,58 @@ jobs: uses: docker/build-push-action@v5 with: context: . -# Only building for AMD64 for now -# platforms: linux/amd64,linux/arm64 + # Only building for AMD64 for now + # platforms: linux/amd64,linux/arm64 push: true - tags: ${{ vars.ECR_REGISTRY }}/${{ vars.ECR_REPOSITORY }}:latest + tags: ${{ vars.ECR_REGISTRY }}/${{ vars.ECR_REPOSITORY }}:${{ github.sha }} - trigger_deploy: - runs-on: ubuntu-latest - needs: [build_test_push] - steps: - - name: Generate App Token - uses: actions/create-github-app-token@v1 - id: app-token - with: - app-id: ${{ vars.DEPLOY_APP_ID }} - private-key: ${{ secrets.DEPLOY_APP_PRIVATE_KEY }} - owner: ${{ github.repository_owner }} - repositories: "appdeploy" + - name: Push Image Digest to SSM + run: | + aws ssm put-parameter \ + --name "/apps/geonetwork4/staging/image_digest" \ + --type "String" \ + --value "$digest" \ + --overwrite + env: + digest: ${{ steps.build_and_push.outputs.digest }} - - name: Trigger Deploy Workflow - uses: actions/github-script@v7 - with: - github-token: ${{ steps.app-token.outputs.token }} - retries: 3 - retry-exempt-status-codes: 204 - script: | - github.rest.actions.createWorkflowDispatch({ - owner: 'aodn', - repo: 'appdeploy', - workflow_id: 'deploy.yml', - ref: 'main', - inputs: { - app_name: 'geonetwork4', - environment: 'staging', - image_tag: '${{ needs.build_test_push.outputs.image_digest }}', - } - }) + +# Optional deployment job if you want to update the task immediately +# However, the appdeploy repository is configured to run drift jobs on an hourly basis +# See: https://terrateam.io/docs/features/drift-detection + +# deploy: +# runs-on: ubuntu-latest +# environment: staging +# needs: [build_push] +# steps: +# - name: Configure AWS Credentials +# uses: aws-actions/configure-aws-credentials@v4 +# with: +# audience: sts.amazonaws.com +# aws-region: ${{ vars.AWS_REGION }} +# role-to-assume: ${{ secrets.AWS_ROLE_ARN }} +# +# - name: Get Currently Running Task Definition +# id: get-current-task-definition +# run: | +# aws ecs describe-task-definition \ +# --task-definition ${{ vars.FAMILY }} \ +# --query taskDefinition > task-definition.json +# +# - name: Update Task Definition with Image Digest +# id: update-api-image-tag +# uses: aws-actions/amazon-ecs-render-task-definition@v1 +# with: +# task-definition: task-definition.json +# container-name: app +# image: ${{ vars.ECR_REGISTRY }}/${{ vars.ECR_REPOSITORY }}@${{ needs.build_push.outputs.image_digest }} +# +# - name: Deploy to Amazon ECS service +# uses: aws-actions/amazon-ecs-deploy-task-definition@v1 +# with: +# task-definition: ${{ steps.update-api-image-tag.outputs.task-definition }} +# service: ${{ vars.FAMILY }} +# cluster: ${{ vars.CLUSTER }} +# force-new-deployment: true +# wait-for-service-stability: true From 639079681435a9fe87744eb3796f47009b8dfb36 Mon Sep 17 00:00:00 2001 From: Stefan Hattrell <29941279+digorgonzola@users.noreply.github.com> Date: Wed, 24 Jan 2024 12:47:46 +1100 Subject: [PATCH 43/51] github builds: remove unncessary extra build no longer needed since we are not running tests --- .github/workflows/build-production.yml | 7 ------- .github/workflows/build-staging.yml | 7 ------- 2 files changed, 14 deletions(-) diff --git a/.github/workflows/build-production.yml b/.github/workflows/build-production.yml index d0c0998..143cef2 100644 --- a/.github/workflows/build-production.yml +++ b/.github/workflows/build-production.yml @@ -35,13 +35,6 @@ jobs: - name: Build with Maven run: mvn -B package --file pom.xml - - name: Build Docker Image - uses: docker/build-push-action@v5 - with: - context: . - load: true - tags: ${{ vars.ECR_REPOSITORY }}:${{ github.ref_name }} - - name: Configure AWS Credentials uses: aws-actions/configure-aws-credentials@v4 with: diff --git a/.github/workflows/build-staging.yml b/.github/workflows/build-staging.yml index d3d27a9..1008e03 100644 --- a/.github/workflows/build-staging.yml +++ b/.github/workflows/build-staging.yml @@ -42,13 +42,6 @@ jobs: - name: Build with Maven run: mvn -B package --file pom.xml - - name: Build Docker Image - uses: docker/build-push-action@v5 - with: - context: . - load: true - tags: ${{ vars.ECR_REPOSITORY }}:${{ github.sha }} - - name: Configure AWS Credentials uses: aws-actions/configure-aws-credentials@v4 with: From 8051b6dcb1238ffd665e4f8e4dc27b23e944faa9 Mon Sep 17 00:00:00 2001 From: Stefan Hattrell <29941279+digorgonzola@users.noreply.github.com> Date: Wed, 24 Jan 2024 13:08:43 +1100 Subject: [PATCH 44/51] github: add .env files for environment variables --- .github/environment/production.env | 3 +++ .github/environment/staging.env | 3 +++ 2 files changed, 6 insertions(+) create mode 100644 .github/environment/production.env create mode 100644 .github/environment/staging.env diff --git a/.github/environment/production.env b/.github/environment/production.env new file mode 100644 index 0000000..cf20cb0 --- /dev/null +++ b/.github/environment/production.env @@ -0,0 +1,3 @@ +AWS_REGION=ap-southeast-2 +ECR_REGISTRY=450356697252.dkr.ecr.ap-southeast-2.amazonaws.com +ECR_REPOSITORY=geonetwork4 diff --git a/.github/environment/staging.env b/.github/environment/staging.env new file mode 100644 index 0000000..cf20cb0 --- /dev/null +++ b/.github/environment/staging.env @@ -0,0 +1,3 @@ +AWS_REGION=ap-southeast-2 +ECR_REGISTRY=450356697252.dkr.ecr.ap-southeast-2.amazonaws.com +ECR_REPOSITORY=geonetwork4 From 843f454174f541cda30433caaad464563a9b65f4 Mon Sep 17 00:00:00 2001 From: Stefan Hattrell <29941279+digorgonzola@users.noreply.github.com> Date: Wed, 24 Jan 2024 13:13:38 +1100 Subject: [PATCH 45/51] add README.md for github variable .env files --- .github/environment/README.md | 13 +++++++++++++ 1 file changed, 13 insertions(+) create mode 100644 .github/environment/README.md diff --git a/.github/environment/README.md b/.github/environment/README.md new file mode 100644 index 0000000..25658a3 --- /dev/null +++ b/.github/environment/README.md @@ -0,0 +1,13 @@ +## Github Deployment Environments +Github deployment environments are used to define unique settings for each environment i.e. staging and production + +The build and push workflows need to know which AWS account to push updated docker images to. + +### DotEnv Files +The .env files in this directory are here as a record of the "variables" and their values. + +The variables can be updated from these files using the following command: +```bash +gh variable set -R aodn/ -e -f .env + +``` From 9b36555e88455b1e5d7bfcf48d0d873972eb333f Mon Sep 17 00:00:00 2001 From: Stefan Hattrell <29941279+digorgonzola@users.noreply.github.com> Date: Wed, 24 Jan 2024 13:29:13 +1100 Subject: [PATCH 46/51] test.yml: make docker build steps more generic --- .github/workflows/test.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 5772e65..3201c5e 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -59,8 +59,8 @@ jobs: with: context: . load: true - tags: ${{ vars.ECR_REPOSITORY }}:${{ steps.set_image_tag.outputs.image_tag }} + tags: image:${{ github.sha }} - name: Test Docker Image run: | - container-structure-test test --image ${{ vars.ECR_REPOSITORY }}:${{ steps.set_image_tag.outputs.image_tag }} --config tests/config.yaml + container-structure-test test --image image:${{ github.sha }} --config tests/config.yaml From 27d8c80ec01fa1dfa69872e60f1905fc00f88782 Mon Sep 17 00:00:00 2001 From: Stefan Hattrell <29941279+digorgonzola@users.noreply.github.com> Date: Wed, 24 Jan 2024 13:32:37 +1100 Subject: [PATCH 47/51] test.yml: rename job to "build_test" --- .github/workflows/test.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 3201c5e..1d6cbdc 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -15,7 +15,7 @@ permissions: contents: read jobs: - build_test_push: + build_test: runs-on: ubuntu-latest steps: - name: Checkout From a48ee1b620c70a91cb528ebca66b9a736909ec4f Mon Sep 17 00:00:00 2001 From: Stefan Hattrell <29941279+digorgonzola@users.noreply.github.com> Date: Wed, 24 Jan 2024 13:38:38 +1100 Subject: [PATCH 48/51] github: ignore changes to .md files --- .github/workflows/build-staging.yml | 4 ++++ .github/workflows/test.yml | 2 ++ 2 files changed, 6 insertions(+) diff --git a/.github/workflows/build-staging.yml b/.github/workflows/build-staging.yml index 1008e03..8b6f878 100644 --- a/.github/workflows/build-staging.yml +++ b/.github/workflows/build-staging.yml @@ -5,12 +5,16 @@ on: branches: - master - main + paths-ignore: + - '**/*.md' # This is only here for testing purposes while in a PR #TODO remove before merge to main pull_request: branches: - master - main + paths-ignore: + - '**/*.md' permissions: id-token: write diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 1d6cbdc..5f1819f 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -5,6 +5,8 @@ on: branches: - master - main + paths-ignore: + - '**/*.md' concurrency: group: ${{ github.ref }} From c27e7b5b2d74077a5cc5ddec3b5f571ab00220a6 Mon Sep 17 00:00:00 2001 From: Stefan Hattrell <29941279+digorgonzola@users.noreply.github.com> Date: Wed, 24 Jan 2024 13:41:05 +1100 Subject: [PATCH 49/51] github: further paths to ignore --- .github/workflows/build-staging.yml | 2 ++ .github/workflows/test.yml | 1 + 2 files changed, 3 insertions(+) diff --git a/.github/workflows/build-staging.yml b/.github/workflows/build-staging.yml index 8b6f878..6761816 100644 --- a/.github/workflows/build-staging.yml +++ b/.github/workflows/build-staging.yml @@ -7,6 +7,7 @@ on: - main paths-ignore: - '**/*.md' + - '.github/environment/**' # This is only here for testing purposes while in a PR #TODO remove before merge to main pull_request: @@ -15,6 +16,7 @@ on: - main paths-ignore: - '**/*.md' + - '.github/environment/**' permissions: id-token: write diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 5f1819f..17144b0 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -7,6 +7,7 @@ on: - main paths-ignore: - '**/*.md' + - '.github/environment/**' concurrency: group: ${{ github.ref }} From 74b0110d39d7ce9a210aa9b8de9d48c54769478a Mon Sep 17 00:00:00 2001 From: Stefan Hattrell <29941279+digorgonzola@users.noreply.github.com> Date: Wed, 7 Feb 2024 11:33:26 +1100 Subject: [PATCH 50/51] github: update workflows with deploy trigger --- .github/workflows/build-production.yml | 72 ++++++++++++------------- .github/workflows/build-staging.yml | 75 ++++++++++++-------------- 2 files changed, 66 insertions(+), 81 deletions(-) diff --git a/.github/workflows/build-production.yml b/.github/workflows/build-production.yml index 143cef2..fc56bff 100644 --- a/.github/workflows/build-production.yml +++ b/.github/workflows/build-production.yml @@ -9,6 +9,9 @@ permissions: id-token: write contents: read +env: + environment_name: production + jobs: build_push: runs-on: ubuntu-latest @@ -60,49 +63,40 @@ jobs: - name: Push Image Digest to SSM run: | aws ssm put-parameter \ - --name "/apps/sample-django-app/production/image_digest" \ + --name "/apps/sample-django-app/${{ env.environment_name }}/image_digest" \ --type "String" \ --value "$digest" \ --overwrite env: digest: ${{ steps.build_and_push.outputs.digest }} -# Optional deployment job if you want to update the task immediately -# However, the appdeploy repository is configured to run drift jobs on an hourly basis -# See: https://terrateam.io/docs/features/drift-detection + trigger_deploy: + runs-on: ubuntu-latest + needs: [build_push] + steps: + - name: Generate App Token + uses: actions/create-github-app-token@v1 + id: app-token + with: + app-id: ${{ vars.DEPLOY_APP_ID }} + private-key: ${{ secrets.DEPLOY_APP_PRIVATE_KEY }} + owner: ${{ github.repository_owner }} + repositories: "appdeploy" -# deploy: -# runs-on: ubuntu-latest -# environment: production -# needs: [build_push] -# steps: -# - name: Configure AWS Credentials -# uses: aws-actions/configure-aws-credentials@v4 -# with: -# audience: sts.amazonaws.com -# aws-region: ${{ vars.AWS_REGION }} -# role-to-assume: ${{ secrets.AWS_ROLE_ARN }} -# -# - name: Get Currently Running Task Definition -# id: get-current-task-definition -# run: | -# aws ecs describe-task-definition \ -# --task-definition ${{ vars.FAMILY }} \ -# --query taskDefinition > task-definition.json -# -# - name: Update Task Definition with Image Digest -# id: update-api-image-tag -# uses: aws-actions/amazon-ecs-render-task-definition@v1 -# with: -# task-definition: task-definition.json -# container-name: app -# image: ${{ vars.ECR_REGISTRY }}/${{ vars.ECR_REPOSITORY }}@${{ needs.build_push.outputs.image_digest }} -# -# - name: Deploy to Amazon ECS service -# uses: aws-actions/amazon-ecs-deploy-task-definition@v1 -# with: -# task-definition: ${{ steps.update-api-image-tag.outputs.task-definition }} -# service: ${{ vars.FAMILY }} -# cluster: ${{ vars.CLUSTER }} -# force-new-deployment: true -# wait-for-service-stability: true + - name: Trigger Deploy Workflow + uses: actions/github-script@v7 + with: + github-token: ${{ steps.app-token.outputs.token }} + retries: 3 + retry-exempt-status-codes: 204 + script: | + github.rest.actions.createWorkflowDispatch({ + owner: 'aodn', + repo: 'appdeploy', + workflow_id: 'deploy.yml', + ref: 'main', + inputs: { + app_name: 'geonetwork4', + environment: '${{ env.environment_name }}' + } + }) diff --git a/.github/workflows/build-staging.yml b/.github/workflows/build-staging.yml index 6761816..19ccb5c 100644 --- a/.github/workflows/build-staging.yml +++ b/.github/workflows/build-staging.yml @@ -22,12 +22,13 @@ permissions: id-token: write contents: read +env: + environment_name: staging + jobs: build_push: runs-on: ubuntu-latest environment: staging - outputs: - image_digest: ${{ steps.build_and_push.outputs.digest }} steps: - name: Checkout uses: actions/checkout@v4 @@ -73,50 +74,40 @@ jobs: - name: Push Image Digest to SSM run: | aws ssm put-parameter \ - --name "/apps/geonetwork4/staging/image_digest" \ + --name "/apps/geonetwork4/${{ env.environment_name }}/image_digest" \ --type "String" \ --value "$digest" \ --overwrite env: digest: ${{ steps.build_and_push.outputs.digest }} + trigger_deploy: + runs-on: ubuntu-latest + needs: [build_push] + steps: + - name: Generate App Token + uses: actions/create-github-app-token@v1 + id: app-token + with: + app-id: ${{ vars.DEPLOY_APP_ID }} + private-key: ${{ secrets.DEPLOY_APP_PRIVATE_KEY }} + owner: ${{ github.repository_owner }} + repositories: "appdeploy" -# Optional deployment job if you want to update the task immediately -# However, the appdeploy repository is configured to run drift jobs on an hourly basis -# See: https://terrateam.io/docs/features/drift-detection - -# deploy: -# runs-on: ubuntu-latest -# environment: staging -# needs: [build_push] -# steps: -# - name: Configure AWS Credentials -# uses: aws-actions/configure-aws-credentials@v4 -# with: -# audience: sts.amazonaws.com -# aws-region: ${{ vars.AWS_REGION }} -# role-to-assume: ${{ secrets.AWS_ROLE_ARN }} -# -# - name: Get Currently Running Task Definition -# id: get-current-task-definition -# run: | -# aws ecs describe-task-definition \ -# --task-definition ${{ vars.FAMILY }} \ -# --query taskDefinition > task-definition.json -# -# - name: Update Task Definition with Image Digest -# id: update-api-image-tag -# uses: aws-actions/amazon-ecs-render-task-definition@v1 -# with: -# task-definition: task-definition.json -# container-name: app -# image: ${{ vars.ECR_REGISTRY }}/${{ vars.ECR_REPOSITORY }}@${{ needs.build_push.outputs.image_digest }} -# -# - name: Deploy to Amazon ECS service -# uses: aws-actions/amazon-ecs-deploy-task-definition@v1 -# with: -# task-definition: ${{ steps.update-api-image-tag.outputs.task-definition }} -# service: ${{ vars.FAMILY }} -# cluster: ${{ vars.CLUSTER }} -# force-new-deployment: true -# wait-for-service-stability: true + - name: Trigger Deploy Workflow + uses: actions/github-script@v7 + with: + github-token: ${{ steps.app-token.outputs.token }} + retries: 3 + retry-exempt-status-codes: 204 + script: | + github.rest.actions.createWorkflowDispatch({ + owner: 'aodn', + repo: 'appdeploy', + workflow_id: 'deploy.yml', + ref: 'main', + inputs: { + app_name: 'geonetwork4', + environment: '${{ env.environment_name }}' + } + }) From c88c1a6e7eb27db3a2530f28941717e17312e134 Mon Sep 17 00:00:00 2001 From: Stefan Hattrell <29941279+digorgonzola@users.noreply.github.com> Date: Wed, 7 Feb 2024 11:34:33 +1100 Subject: [PATCH 51/51] Revert "update .gitignore with tf/tg rules" This reverts commit 073fe6a28fb1cffeb4fe6a2e7d4220dba81bdda7. --- .gitignore | 10 +--------- 1 file changed, 1 insertion(+), 9 deletions(-) diff --git a/.gitignore b/.gitignore index 89195d6..91b83ad 100644 --- a/.gitignore +++ b/.gitignore @@ -26,12 +26,4 @@ replay_pid* # Ingore local generated folders elasticdata/ gn4_data/ -.env - -# Terraform/Terragrunt directories and files -**/.terraform/* -**/.terragrunt-cache/ -*.tfstate -*.tfstate.* -terragrunt-debug.tfvars.json -terragrunt_rendered.json +.env \ No newline at end of file