feat: DBTP-914 - Environment pipeline terraform apply (#116)

Co-authored-by: Anthony Roy <[email protected]>
Co-authored-by: Will Gibson <[email protected]>

.checkov.baseline

@@ -6,43 +6,27 @@
                 {
                     "resource": "module.extensions-staging.module.alb.aws_lb.this",
                     "check_ids": [
-                        "CKV2_AWS_20",
                         "CKV2_AWS_28",
                         "CKV_AWS_131",
                         "CKV_AWS_150"
                     ]
                 },
                 {
-                    "resource": "module.extensions-staging.module.alb.aws_lb_listener.alb-listener[\"http\"]",
+                    "resource": "module.extensions-staging.module.alb.aws_lb_listener.alb-listener",
                     "check_ids": [
                         "CKV_AWS_103",
                         "CKV_AWS_2"
                     ]
                 },
-                {
-                    "resource": "module.extensions-staging.module.alb.aws_lb_listener.alb-listener[\"https\"]",
-                    "check_ids": [
-                        "CKV_AWS_103"
-                    ]
-                },
                 {
                     "resource": "module.extensions-staging.module.alb.aws_lb_target_group.http-target-group",
                     "check_ids": [
                         "CKV_AWS_261"
                     ]
                 },
                 {
-                    "resource": "module.extensions-staging.module.alb.aws_security_group.alb-security-group[\"http\"]",
+                    "resource": "module.extensions-staging.module.alb.aws_security_group.alb-security-group",
                     "check_ids": [
-                        "CKV2_AWS_5",
-                        "CKV_AWS_23",
-                        "CKV_AWS_260"
-                    ]
-                },
-                {
-                    "resource": "module.extensions-staging.module.alb.aws_security_group.alb-security-group[\"https\"]",
-                    "check_ids": [
-                        "CKV2_AWS_5",
                         "CKV_AWS_23"
                     ]
                 }
@@ -124,12 +108,6 @@
                     "check_ids": [
                         "CKV_AWS_158"
                     ]
-                },
-                {
-                    "resource": "aws_codebuild_project.environment_pipeline",
-                    "check_ids": [
-                        "CKV_AWS_147"
-                    ]
                 }
             ]
         },
@@ -353,10 +331,10 @@
                     "resource": "aws_vpc.vpc",
                     "check_ids": [
                         "CKV2_AWS_11",
-                        "CKV2_AWS_12"                        
+                        "CKV2_AWS_12"
                     ]
                 }
             ]
         }
     ]
-}
+}

environment-pipelines/buildspec-apply.yml

@@ -0,0 +1,15 @@
+version: 0.2
+phases:
+  install:
+    commands:
+      - export PATH="$CODEBUILD_SRC_DIR/build-tools:$PATH"
+      - mv $CODEBUILD_SRC_DIR_terraform_plan/terraform/${ENVIRONMENT}/plan.tfplan $CODEBUILD_SRC_DIR/terraform/${ENVIRONMENT}/
+  build:
+    commands:
+      - echo "Terraform Apply Phase"
+      - echo "Working on environment ${ENVIRONMENT}"
+      - cd terraform/${ENVIRONMENT}
+      - terraform init
+      - terraform apply plan.tfplan
+artifacts:
+  files: []

environment-pipelines/buildspec-plan.yml

@@ -0,0 +1,23 @@
+version: 0.2
+
+env:
+  exported-variables:
+    - BUILD_ID
+
+phases:
+  install:
+    commands:
+      - export PATH="$CODEBUILD_SRC_DIR/build-tools:$PATH"
+  build:
+    commands:
+      - echo "Terraform Plan Phase"
+      - echo "Working on environment ${ENVIRONMENT}"
+      - cd terraform/${ENVIRONMENT}
+      - terraform init
+      - terraform plan -out=plan.tfplan
+  post_build:
+    commands:
+      - export BUILD_ID="$CODEBUILD_BUILD_ID"
+artifacts:
+  files:
+    - terraform/${ENVIRONMENT}/plan.tfplan

environment-pipelines/buildspec.yml

@@ -19,30 +19,22 @@ phases:
       - COPILOT_VERSION=`cat .copilot-version`
       - PLATFORM_HELPER_VERSION=`cat .platform-helper-version`
       - echo "Install Phase"
-      - yum -y install zip wget which python-pip
+      - yum -y install python-pip
+      - mkdir ./build-tools
+      - cd ./build-tools
       - curl -s -qL -o terraform_install.zip https://releases.hashicorp.com/terraform/${TERRAFORM_VERSION}/terraform_${TERRAFORM_VERSION}_linux_amd64.zip
-      - unzip terraform_install.zip -d /usr/bin/
-      - chmod +x /usr/bin/terraform
-      - VERSION_OUTPUT+=$(terraform version | head -n1)
-      - wget -q https://ecs-cli-v2-release.s3.amazonaws.com/copilot-linux-v${COPILOT_VERSION} -O /usr/bin/copilot
-      - chmod +x /usr/bin/copilot
-      - pip install dbt-platform-helper==$PLATFORM_HELPER_VERSION
-      - VERSION_OUTPUT+="\n$(platform-helper --version)"
-      - VERSION_OUTPUT+="\n$(copilot --version)"
+      - unzip terraform_install.zip
+      - chmod +x terraform
+      - rm terraform_install.zip
+      - curl -s -qL -o copilot https://ecs-cli-v2-release.s3.amazonaws.com/copilot-linux-v${COPILOT_VERSION}
+      - chmod +x copilot
+      - pip install --quiet --target . dbt-platform-helper==$PLATFORM_HELPER_VERSION
+      - VERSION_OUTPUT+="\n$(./terraform --version)"
+      - VERSION_OUTPUT+="\n$(./platform_helper.py --version)"
+      - VERSION_OUTPUT+="\n$(./copilot --version)"
       - VERSION_OUTPUT+="\n$(python3 --version)"
       - VERSION_OUTPUT+="\n$(pip --version)"
       - echo -e "=============\nTool Versions\n-------------\n$VERSION_OUTPUT"
-
-  build:
-    commands:
-      - echo "Terraform Plan Phase"
-      - echo "Working on environment ${ENVIRONMENT}"
-      - cd terraform/${ENVIRONMENT}
-      - terraform init
-      - terraform plan -out=plan.tfplan
-  post_build:
-    commands:
-      - echo "Post Build Phase"
 artifacts:
   files:
-    - terraform/${ENVIRONMENT}/plan.tfplan
+    - "**/*"

environment-pipelines/codebuild.tf

@@ -1,8 +1,9 @@
-resource "aws_codebuild_project" "environment_pipeline" {
-  name          = "${var.application}-environment-pipeline"
-  description   = "Provisions the ${var.application} application's extensions."
-  build_timeout = 5
-  service_role  = aws_iam_role.environment_pipeline_codebuild.arn
+resource "aws_codebuild_project" "environment_pipeline_build" {
+  name           = "${var.application}-environment-pipeline-build"
+  description    = "Provisions the ${var.application} application's extensions."
+  build_timeout  = 5
+  service_role   = aws_iam_role.environment_pipeline_codebuild.arn
+  encryption_key = module.artifact_store.kms_key_arn
 
   artifacts {
     type = "CODEPIPELINE"
@@ -45,3 +46,81 @@ resource "aws_cloudwatch_log_stream" "environment_pipeline_codebuild" {
   name           = "codebuild/${var.application}-environment-terraform/log-stream"
   log_group_name = aws_cloudwatch_log_group.environment_pipeline_codebuild.name
 }
+
+# Terraform plan
+resource "aws_codebuild_project" "environment_pipeline_plan" {
+  name           = "${var.application}-environment-pipeline-plan"
+  description    = "Provisions the ${var.application} application's extensions."
+  build_timeout  = 5
+  service_role   = aws_iam_role.environment_pipeline_codebuild.arn
+  encryption_key = module.artifact_store.kms_key_arn
+
+  artifacts {
+    type = "CODEPIPELINE"
+  }
+
+  cache {
+    type     = "S3"
+    location = module.artifact_store.bucket_name
+  }
+
+  environment {
+    compute_type                = "BUILD_GENERAL1_SMALL"
+    image                       = "aws/codebuild/amazonlinux2-x86_64-standard:5.0"
+    type                        = "LINUX_CONTAINER"
+    image_pull_credentials_type = "CODEBUILD"
+  }
+
+  logs_config {
+    cloudwatch_logs {
+      group_name  = aws_cloudwatch_log_group.environment_pipeline_codebuild.name
+      stream_name = aws_cloudwatch_log_stream.environment_pipeline_codebuild.name
+    }
+  }
+
+  source {
+    type      = "CODEPIPELINE"
+    buildspec = file("${path.module}/buildspec-plan.yml")
+  }
+
+  tags = local.tags
+}
+
+# Terraform apply
+resource "aws_codebuild_project" "environment_pipeline_apply" {
+  name           = "${var.application}-environment-pipeline-apply"
+  description    = "Provisions the ${var.application} application's extensions."
+  build_timeout  = 60
+  service_role   = aws_iam_role.environment_pipeline_codebuild.arn
+  encryption_key = module.artifact_store.kms_key_arn
+
+  artifacts {
+    type = "CODEPIPELINE"
+  }
+
+  cache {
+    type     = "S3"
+    location = module.artifact_store.bucket_name
+  }
+
+  environment {
+    compute_type                = "BUILD_GENERAL1_SMALL"
+    image                       = "aws/codebuild/amazonlinux2-x86_64-standard:5.0"
+    type                        = "LINUX_CONTAINER"
+    image_pull_credentials_type = "CODEBUILD"
+  }
+
+  logs_config {
+    cloudwatch_logs {
+      group_name  = aws_cloudwatch_log_group.environment_pipeline_codebuild.name
+      stream_name = aws_cloudwatch_log_stream.environment_pipeline_codebuild.name
+    }
+  }
+
+  source {
+    type      = "CODEPIPELINE"
+    buildspec = file("${path.module}/buildspec-apply.yml")
+  }
+
+  tags = local.tags
+}

environment-pipelines/codepipeline.tf

@@ -36,31 +36,40 @@ resource "aws_codepipeline" "environment_pipeline" {
     }
   }
 
+  stage {
+    name = "Build"
+
+    action {
+      name             = "InstallTools"
+      category         = "Build"
+      owner            = "AWS"
+      provider         = "CodeBuild"
+      input_artifacts  = ["project_deployment_source"]
+      output_artifacts = ["build_output"]
+      version          = "1"
+
+      configuration = {
+        ProjectName   = "${var.application}-environment-pipeline-build"
+        PrimarySource = "project_deployment_source"
+      }
+    }
+  }
 
   dynamic "stage" {
     for_each = local.stages
     content {
-      name = "Build"
+      name = stage.value.stage_name
 
       action {
-        name             = "InstallTools"
-        category         = "Build"
-        owner            = "AWS"
-        provider         = "CodeBuild"
-        input_artifacts  = ["project_deployment_source"]
-        output_artifacts = ["terraform_plan"]
+        name             = stage.value.name
+        category         = stage.value.category
+        owner            = stage.value.owner
+        provider         = stage.value.provider
+        input_artifacts  = stage.value.input_artifacts
+        output_artifacts = stage.value.output_artifacts
         version          = "1"
-
-        configuration = {
-          ProjectName   = "${var.application}-environment-pipeline"
-          PrimarySource = "project_deployment_source"
-          EnvironmentVariables = jsonencode([
-            {
-              name  = "ENVIRONMENT"
-              value = stage.value.env
-            }
-          ])
-        }
+        configuration    = stage.value.configuration
+        namespace        = stage.value.namespace
       }
     }
   }