Add ingress management

.github/workflows/deploy-dev.yaml

@@ -18,3 +18,5 @@ jobs:
           AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
           AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
           DIGITALOCEAN_ACCESS_TOKEN: ${{ secrets.DIGITALOCEAN_ACCESS_TOKEN }}
+          CERT_MANAGER_ACME_EMAIL: ${{ vars.CERT_MANAGER_ACME_EMAIL }}
+          EXTERNAL_DNS_TOKEN: ${{ secrets.EXTERNAL_DNS_TOKEN }}

main.tf

@@ -12,11 +12,43 @@ terraform {
       source  = "digitalocean/digitalocean"
       version = "~> 2.0"
     }
+
+    helm = {
+      source  = "hashicorp/helm"
+      version = "2.12.1"
+    }
+
+    kubernetes = {
+      source  = "hashicorp/kubernetes"
+      version = "2.25.2"
+    }
   }
 }
 
 provider "digitalocean" {}
 
+provider "helm" {
+  kubernetes {
+    host  = digitalocean_kubernetes_cluster.this.endpoint
+    token = digitalocean_kubernetes_cluster.this.kube_config[0].token
+    cluster_ca_certificate = base64decode(
+      digitalocean_kubernetes_cluster.this.kube_config[0].cluster_ca_certificate
+    )
+  }
+
+  experiments {
+    manifest = true
+  }
+}
+
+provider "kubernetes" {
+  host  = digitalocean_kubernetes_cluster.this.endpoint
+  token = digitalocean_kubernetes_cluster.this.kube_config[0].token
+  cluster_ca_certificate = base64decode(
+    digitalocean_kubernetes_cluster.this.kube_config[0].cluster_ca_certificate
+  )
+}
+
 variable "environment" {
   type = string
 }
@@ -42,11 +74,19 @@ variable "kubernetes_default_node_pool_node_count" {
   type = number
 }
 
+variable "cert_manager_acme_email" {
+  type = string
+}
+
+variable "external_dns_token" {
+  type      = string
+  sensitive = true
+}
+
 output "host" {
   value     = digitalocean_kubernetes_cluster.this.kube_config[0].host
   sensitive = true
 }
-
 output "token" {
   value     = digitalocean_kubernetes_cluster.this.kube_config[0].token
   sensitive = true
@@ -68,3 +108,13 @@ resource "digitalocean_kubernetes_cluster" "this" {
     node_count = var.kubernetes_default_node_pool_node_count
   }
 }
+
+module "runtime" {
+  source = "./runtime"
+
+  environment             = var.environment
+  service                 = var.service
+  cluster_name            = digitalocean_kubernetes_cluster.this.name
+  cert_manager_acme_email = var.cert_manager_acme_email
+  external_dns_token      = var.external_dns_token
+}

runtime/cert_manager.tf

@@ -0,0 +1,58 @@
+variable "cert_manager_acme_email" {
+  type = string
+}
+
+resource "helm_release" "cert_manager" {
+  name       = "cert-manager"
+  chart      = "cert-manager"
+  repository = "https://charts.jetstack.io"
+  version    = "v1.14.2"
+
+  namespace       = kubernetes_namespace.this.metadata[0].name
+  atomic          = true
+  cleanup_on_fail = true
+  reset_values    = true
+
+  set {
+    name  = "installCRDs"
+    value = true
+  }
+}
+
+resource "kubernetes_manifest" "cert_manager_issuer_staging" {
+  manifest = yamldecode(<<-EOT
+    apiVersion: cert-manager.io/v1
+    kind: ClusterIssuer
+    metadata:
+      name: letsencrypt-staging
+    spec:
+      acme:
+        email: ${var.cert_manager_acme_email}
+        server: https://acme-staging-v02.api.letsencrypt.org/directory
+        privateKeySecretRef:
+          name: letsencrypt-staging-key
+        solvers:
+        - http01:
+            ingress: {}
+  EOT
+  )
+}
+
+resource "kubernetes_manifest" "cert_manager_issuer_prod" {
+  manifest = yamldecode(<<-EOT
+    apiVersion: cert-manager.io/v1
+    kind: ClusterIssuer
+    metadata:
+      name: letsencrypt-prod
+    spec:
+      acme:
+        email: ${var.cert_manager_acme_email}
+        server: https://acme-v02.api.letsencrypt.org/directory
+        privateKeySecretRef:
+          name: letsencrypt-prod-key
+        solvers:
+        - http01:
+            ingress: {}
+  EOT
+  )
+}

runtime/external_dns.tf

@@ -0,0 +1,68 @@
+variable "external_dns_token" {
+  type      = string
+  sensitive = true
+}
+
+resource "kubernetes_secret" "external_dns_token" {
+  metadata {
+    generate_name = "external-dns-token-"
+    namespace     = kubernetes_namespace.this.metadata[0].name
+  }
+
+  data = {
+    token = var.external_dns_token
+  }
+}
+
+resource "helm_release" "external_dns" {
+  name       = "external-dns"
+  chart      = "external-dns"
+  repository = "https://kubernetes-sigs.github.io/external-dns/"
+  version    = "1.14.3"
+
+  namespace       = kubernetes_namespace.this.metadata[0].name
+  atomic          = true
+  cleanup_on_fail = true
+  reset_values    = true
+
+  set {
+    name  = "policy"
+    value = "sync"
+  }
+
+  set {
+    name  = "txtPrefix"
+    value = "runtime-dev-external-dns-"
+  }
+
+  set {
+    name  = "txtOwnerId"
+    value = data.digitalocean_kubernetes_cluster.cluster.id
+  }
+
+  set_list {
+    name  = "sources"
+    value = ["ingress"]
+  }
+
+  set {
+    name  = "provider.name"
+    value = "digitalocean"
+  }
+
+  values = [
+    yamlencode({
+      env = [
+        {
+          name = "DO_TOKEN"
+          valueFrom = {
+            secretKeyRef = {
+              name = kubernetes_secret.external_dns_token.metadata[0].name
+              key  = "token"
+            }
+          }
+        }
+      ]
+    })
+  ]
+}

runtime/ingress_nginx.tf

@@ -0,0 +1,17 @@
+resource "helm_release" "ingress_nginx" {
+  name       = "ingress-nginx"
+  chart      = "ingress-nginx"
+  repository = "https://kubernetes.github.io/ingress-nginx"
+  version    = "4.9.1"
+
+  namespace       = kubernetes_namespace.this.metadata[0].name
+  atomic          = true
+  cleanup_on_fail = true
+  reset_values    = true
+
+  set {
+    name  = "controller.ingressClassResource.default"
+    value = true
+  }
+}
+

runtime/main.tf

@@ -0,0 +1,40 @@
+terraform {
+  required_providers {
+    digitalocean = {
+      source  = "digitalocean/digitalocean"
+      version = "~> 2.0"
+    }
+
+    helm = {
+      source  = "hashicorp/helm"
+      version = "2.12.1"
+    }
+
+    kubernetes = {
+      source  = "hashicorp/kubernetes"
+      version = "2.25.2"
+    }
+  }
+}
+
+variable "environment" {
+  type = string
+}
+
+variable "service" {
+  type = string
+}
+
+variable "cluster_name" {
+  type = string
+}
+
+data "digitalocean_kubernetes_cluster" "cluster" {
+  name = var.cluster_name
+}
+
+resource "kubernetes_namespace" "this" {
+  metadata {
+    name = var.service
+  }
+}

terraform-env.sh

@@ -15,6 +15,15 @@ shift
 environment="$1"
 shift
 
+if [[ -z "${EXTERNAL_DNS_TOKEN+x}" ]]; then
+  echo "Missing required environment variable: EXTERNAL_DNS_TOKEN" >&2
+  exit 1
+fi
+
+if [[ -z "${CERT_MANAGER_ACME_EMAIL+x}" ]]; then
+  echo "Missing required environment variable: CERT_MANAGER_ACME_EMAIL" >&2
+  exit 1
+fi
 
 stateBucket="do-foundations-$environment-terraform"
 stateKey="$service/$environment.tfstate"
@@ -25,10 +34,18 @@ tfCliArgsInit=(
   "-backend-config=key=$stateKey"
 )
 
-tfCliArgsApply=(
+tfCliArgsPlan=(
   "-var=environment=$environment"
+  "-var=external_dns_token=$EXTERNAL_DNS_TOKEN"
+  "-var=cert_manager_acme_email=$CERT_MANAGER_ACME_EMAIL"
   "-var-file=$environment.tfvars"
 )
 
+# shellcheck disable=SC2206
+tfCliArgsApply=(
+  ${tfCliArgsPlan[@]}
+)
+
 echo "export TF_CLI_ARGS_init='${tfCliArgsInit[*]}'"
+echo "export TF_CLI_ARGS_plan='${tfCliArgsPlan[*]}'"
 echo "export TF_CLI_ARGS_apply='${tfCliArgsApply[*]}'"