From ae44a9fc9b38937618e9b94654168452b56e0931 Mon Sep 17 00:00:00 2001 From: Chris Connelly Date: Thu, 8 Feb 2024 17:14:31 +0000 Subject: [PATCH 1/3] feat: add `runtime` module with ingress controller This uses the fully open `ingress-nginx` controller. --- .terraform.lock.hcl | 46 ++++++++++++++++++++++++++++++++++++++++ main.tf | 40 ++++++++++++++++++++++++++++++++++ runtime/ingress_nginx.tf | 17 +++++++++++++++ runtime/main.tf | 40 ++++++++++++++++++++++++++++++++++ 4 files changed, 143 insertions(+) create mode 100644 runtime/ingress_nginx.tf create mode 100644 runtime/main.tf diff --git a/.terraform.lock.hcl b/.terraform.lock.hcl index dcbb5c4..c878969 100644 --- a/.terraform.lock.hcl +++ b/.terraform.lock.hcl @@ -27,3 +27,49 @@ provider "registry.terraform.io/digitalocean/digitalocean" { "zh:fbe5edf5337adb7360f9ffef57d02b397555b6a89bba68d1b60edfec6e23f02c", ] } + +provider "registry.terraform.io/hashicorp/helm" { + version = "2.12.1" + constraints = "2.12.1" + hashes = [ + "h1:7wfYOAeSEchHB8idNl+2jf+OkFi9zFSOLWkEZFuTCik=", + "h1:aBfcqM4cbywa7TAxfT1YoFS+Cst9waerlm4XErFmJlk=", + "h1:sgYI7lwGqJqPopY3NGmhb1eQ0YbH8PIXaAZAmnJrAvw=", + "h1:sjzfyNQAjtF9zXHxB67geryjGkHaPDMMVw9iqPP5pkE=", + "zh:1d623fb1662703f2feb7860e3c795d849c77640eecbc5a776784d08807b15004", + "zh:253a5bc62ba2c4314875139e3fbd2feaad5ef6b0fb420302a474ab49e8e51a38", + "zh:282358f4ad4f20d0ccaab670b8645228bfad1c03ac0d0df5889f0aea8aeac01a", + "zh:4fd06af3091a382b3f0d8f0a60880f59640d2b6d9d6a31f9a873c6f1bde1ec50", + "zh:6816976b1830f5629ae279569175e88b497abbbac30ee809948a1f923c67a80d", + "zh:7d82c4150cdbf48cfeec867be94c7b9bd7682474d4df0ebb7e24e148f964844f", + "zh:83f062049eea2513118a4c6054fb06c8600bac96196f25aed2cc21898ec86e93", + "zh:a79eec0cf4c08fca79e44033ec6e470f25ff23c3e2c7f9bc707ed7771c1072c0", + "zh:b2b2d904b2821a6e579910320605bc478bbef063579a23fbfdd6fcb5871b81f8", + "zh:e91177ca06a15487fc570cb81ecef6359aa399459ea2aa7c4f7367ba86f6fcad", + "zh:e976bcb82996fc4968f8382bbcb6673efb1f586bf92074058a232028d97825b1", + "zh:f569b65999264a9416862bca5cd2a6177d94ccb0424f3a4ef424428912b9cb3c", + ] +} + +provider "registry.terraform.io/hashicorp/kubernetes" { + version = "2.25.2" + constraints = "2.25.2" + hashes = [ + "h1:+Yi+ho+dpgEmMz6Mt/9O/kDQw9HTcrLWkMyTBFN9yIE=", + "h1:QlTKoO0efmkzgX/9y0DQCEkg7VeidOSQW8epF6B4cEQ=", + "h1:T1WAQt40cAk721H0AM/eZ5YuodJaIfS8r3Tu7rKCJJE=", + "h1:o/+UcYEaEHrQzq2kkWw2MohCK033u6vY+T6cmHd46QU=", + "zh:044788ac936e0e8ece8f78a2e4e366ecd435ea8235388eaf2cbc8e7975d9d970", + "zh:24f5ff01df91f51f00ee7ff39430adeb63bb2ca4ea0042e68f06d6b65808c02f", + "zh:49984aa0aa1faa8c4f01e8faa039322f1e6fdaeab0b7e32f5c6e96edfde36a38", + "zh:4eeceaff56bac9fc782e7e33f157fa2c7e9a47b2c3c3d12da2642c312ace73f6", + "zh:4f49b6419345960d5af475e0200c243af4c9c140b0ee64799fe1fc9b023c49ea", + "zh:7958414d516867a2263a978792a24843f80023fb233cf051ff4095adc9803d85", + "zh:c633a755fc95e9ff0cd73656f052947afd85883a0987dde5198113aa48474156", + "zh:cbfe958d119795004ce1e8001449d01c056fa2a062b51d07843d98be216337d7", + "zh:cfb85392e18768578d4c943438897083895719be678227fd90efbe3500702a56", + "zh:d705a661ed5da425dd236a48645bec39fe78a67d2e70e8460b720417cbf260ac", + "zh:ddd7a01263da3793df4f3b5af65f166307eed5acf525e51e058cda59009cc856", + "zh:f569b65999264a9416862bca5cd2a6177d94ccb0424f3a4ef424428912b9cb3c", + ] +} diff --git a/main.tf b/main.tf index 01bdf6f..64bb265 100644 --- a/main.tf +++ b/main.tf @@ -12,11 +12,43 @@ terraform { source = "digitalocean/digitalocean" version = "~> 2.0" } + + helm = { + source = "hashicorp/helm" + version = "2.12.1" + } + + kubernetes = { + source = "hashicorp/kubernetes" + version = "2.25.2" + } } } provider "digitalocean" {} +provider "helm" { + kubernetes { + host = digitalocean_kubernetes_cluster.this.endpoint + token = digitalocean_kubernetes_cluster.this.kube_config[0].token + cluster_ca_certificate = base64decode( + digitalocean_kubernetes_cluster.this.kube_config[0].cluster_ca_certificate + ) + } + + experiments { + manifest = true + } +} + +provider "kubernetes" { + host = digitalocean_kubernetes_cluster.this.endpoint + token = digitalocean_kubernetes_cluster.this.kube_config[0].token + cluster_ca_certificate = base64decode( + digitalocean_kubernetes_cluster.this.kube_config[0].cluster_ca_certificate + ) +} + variable "environment" { type = string } @@ -68,3 +100,11 @@ resource "digitalocean_kubernetes_cluster" "this" { node_count = var.kubernetes_default_node_pool_node_count } } + +module "runtime" { + source = "./runtime" + + environment = var.environment + service = var.service + cluster_name = digitalocean_kubernetes_cluster.this.name +} diff --git a/runtime/ingress_nginx.tf b/runtime/ingress_nginx.tf new file mode 100644 index 0000000..b3c5928 --- /dev/null +++ b/runtime/ingress_nginx.tf @@ -0,0 +1,17 @@ +resource "helm_release" "ingress_nginx" { + name = "ingress-nginx" + chart = "ingress-nginx" + repository = "https://kubernetes.github.io/ingress-nginx" + version = "4.9.1" + + namespace = kubernetes_namespace.this.metadata[0].name + atomic = true + cleanup_on_fail = true + reset_values = true + + set { + name = "controller.ingressClassResource.default" + value = true + } +} + diff --git a/runtime/main.tf b/runtime/main.tf new file mode 100644 index 0000000..080c2f9 --- /dev/null +++ b/runtime/main.tf @@ -0,0 +1,40 @@ +terraform { + required_providers { + digitalocean = { + source = "digitalocean/digitalocean" + version = "~> 2.0" + } + + helm = { + source = "hashicorp/helm" + version = "2.12.1" + } + + kubernetes = { + source = "hashicorp/kubernetes" + version = "2.25.2" + } + } +} + +variable "environment" { + type = string +} + +variable "service" { + type = string +} + +variable "cluster_name" { + type = string +} + +data "digitalocean_kubernetes_cluster" "cluster" { + name = var.cluster_name +} + +resource "kubernetes_namespace" "this" { + metadata { + name = var.service + } +} From 8bdce9476c8a25e7850068edce0d1f20c016d471 Mon Sep 17 00:00:00 2001 From: Chris Connelly Date: Thu, 8 Feb 2024 18:45:18 +0000 Subject: [PATCH 2/3] feat: add `external-dns` to `runtime` --- .github/workflows/deploy-dev.yaml | 1 + main.tf | 13 ++++-- runtime/external_dns.tf | 68 +++++++++++++++++++++++++++++++ terraform-env.sh | 13 +++++- 4 files changed, 90 insertions(+), 5 deletions(-) create mode 100644 runtime/external_dns.tf diff --git a/.github/workflows/deploy-dev.yaml b/.github/workflows/deploy-dev.yaml index b129de5..43e893b 100644 --- a/.github/workflows/deploy-dev.yaml +++ b/.github/workflows/deploy-dev.yaml @@ -18,3 +18,4 @@ jobs: AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} DIGITALOCEAN_ACCESS_TOKEN: ${{ secrets.DIGITALOCEAN_ACCESS_TOKEN }} + EXTERNAL_DNS_TOKEN: ${{ secrets.EXTERNAL_DNS_TOKEN }} diff --git a/main.tf b/main.tf index 64bb265..ff9875d 100644 --- a/main.tf +++ b/main.tf @@ -74,11 +74,15 @@ variable "kubernetes_default_node_pool_node_count" { type = number } +variable "external_dns_token" { + type = string + sensitive = true +} + output "host" { value = digitalocean_kubernetes_cluster.this.kube_config[0].host sensitive = true } - output "token" { value = digitalocean_kubernetes_cluster.this.kube_config[0].token sensitive = true @@ -104,7 +108,8 @@ resource "digitalocean_kubernetes_cluster" "this" { module "runtime" { source = "./runtime" - environment = var.environment - service = var.service - cluster_name = digitalocean_kubernetes_cluster.this.name + environment = var.environment + service = var.service + cluster_name = digitalocean_kubernetes_cluster.this.name + external_dns_token = var.external_dns_token } diff --git a/runtime/external_dns.tf b/runtime/external_dns.tf new file mode 100644 index 0000000..b87cd2d --- /dev/null +++ b/runtime/external_dns.tf @@ -0,0 +1,68 @@ +variable "external_dns_token" { + type = string + sensitive = true +} + +resource "kubernetes_secret" "external_dns_token" { + metadata { + generate_name = "external-dns-token-" + namespace = kubernetes_namespace.this.metadata[0].name + } + + data = { + token = var.external_dns_token + } +} + +resource "helm_release" "external_dns" { + name = "external-dns" + chart = "external-dns" + repository = "https://kubernetes-sigs.github.io/external-dns/" + version = "1.14.3" + + namespace = kubernetes_namespace.this.metadata[0].name + atomic = true + cleanup_on_fail = true + reset_values = true + + set { + name = "policy" + value = "sync" + } + + set { + name = "txtPrefix" + value = "runtime-dev-external-dns-" + } + + set { + name = "txtOwnerId" + value = data.digitalocean_kubernetes_cluster.cluster.id + } + + set_list { + name = "sources" + value = ["ingress"] + } + + set { + name = "provider.name" + value = "digitalocean" + } + + values = [ + yamlencode({ + env = [ + { + name = "DO_TOKEN" + valueFrom = { + secretKeyRef = { + name = kubernetes_secret.external_dns_token.metadata[0].name + key = "token" + } + } + } + ] + }) + ] +} diff --git a/terraform-env.sh b/terraform-env.sh index f38f1d6..a8c7fc0 100755 --- a/terraform-env.sh +++ b/terraform-env.sh @@ -15,6 +15,10 @@ shift environment="$1" shift +if [[ -z "${EXTERNAL_DNS_TOKEN+x}" ]]; then + echo "Missing required environment variable: EXTERNAL_DNS_TOKEN" >&2 + exit 1 +fi stateBucket="do-foundations-$environment-terraform" stateKey="$service/$environment.tfstate" @@ -25,10 +29,17 @@ tfCliArgsInit=( "-backend-config=key=$stateKey" ) -tfCliArgsApply=( +tfCliArgsPlan=( "-var=environment=$environment" + "-var=external_dns_token=$EXTERNAL_DNS_TOKEN" "-var-file=$environment.tfvars" ) +# shellcheck disable=SC2206 +tfCliArgsApply=( + ${tfCliArgsPlan[@]} +) + echo "export TF_CLI_ARGS_init='${tfCliArgsInit[*]}'" +echo "export TF_CLI_ARGS_plan='${tfCliArgsPlan[*]}'" echo "export TF_CLI_ARGS_apply='${tfCliArgsApply[*]}'" From 79b6c95ea6ffb6f1394b0072dfe2432d76a39acd Mon Sep 17 00:00:00 2001 From: Chris Connelly Date: Fri, 9 Feb 2024 14:13:05 +0000 Subject: [PATCH 3/3] feat: add `cert-manager` This includes to cluster issuers: `letsencrypt-staging` and `letsencrypt-prod`. --- .github/workflows/deploy-dev.yaml | 1 + main.tf | 13 ++++--- runtime/cert_manager.tf | 58 +++++++++++++++++++++++++++++++ terraform-env.sh | 6 ++++ 4 files changed, 74 insertions(+), 4 deletions(-) create mode 100644 runtime/cert_manager.tf diff --git a/.github/workflows/deploy-dev.yaml b/.github/workflows/deploy-dev.yaml index 43e893b..c9bdeb8 100644 --- a/.github/workflows/deploy-dev.yaml +++ b/.github/workflows/deploy-dev.yaml @@ -18,4 +18,5 @@ jobs: AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} DIGITALOCEAN_ACCESS_TOKEN: ${{ secrets.DIGITALOCEAN_ACCESS_TOKEN }} + CERT_MANAGER_ACME_EMAIL: ${{ vars.CERT_MANAGER_ACME_EMAIL }} EXTERNAL_DNS_TOKEN: ${{ secrets.EXTERNAL_DNS_TOKEN }} diff --git a/main.tf b/main.tf index ff9875d..0462ec3 100644 --- a/main.tf +++ b/main.tf @@ -74,6 +74,10 @@ variable "kubernetes_default_node_pool_node_count" { type = number } +variable "cert_manager_acme_email" { + type = string +} + variable "external_dns_token" { type = string sensitive = true @@ -108,8 +112,9 @@ resource "digitalocean_kubernetes_cluster" "this" { module "runtime" { source = "./runtime" - environment = var.environment - service = var.service - cluster_name = digitalocean_kubernetes_cluster.this.name - external_dns_token = var.external_dns_token + environment = var.environment + service = var.service + cluster_name = digitalocean_kubernetes_cluster.this.name + cert_manager_acme_email = var.cert_manager_acme_email + external_dns_token = var.external_dns_token } diff --git a/runtime/cert_manager.tf b/runtime/cert_manager.tf new file mode 100644 index 0000000..2dc2c4c --- /dev/null +++ b/runtime/cert_manager.tf @@ -0,0 +1,58 @@ +variable "cert_manager_acme_email" { + type = string +} + +resource "helm_release" "cert_manager" { + name = "cert-manager" + chart = "cert-manager" + repository = "https://charts.jetstack.io" + version = "v1.14.2" + + namespace = kubernetes_namespace.this.metadata[0].name + atomic = true + cleanup_on_fail = true + reset_values = true + + set { + name = "installCRDs" + value = true + } +} + +resource "kubernetes_manifest" "cert_manager_issuer_staging" { + manifest = yamldecode(<<-EOT + apiVersion: cert-manager.io/v1 + kind: ClusterIssuer + metadata: + name: letsencrypt-staging + spec: + acme: + email: ${var.cert_manager_acme_email} + server: https://acme-staging-v02.api.letsencrypt.org/directory + privateKeySecretRef: + name: letsencrypt-staging-key + solvers: + - http01: + ingress: {} + EOT + ) +} + +resource "kubernetes_manifest" "cert_manager_issuer_prod" { + manifest = yamldecode(<<-EOT + apiVersion: cert-manager.io/v1 + kind: ClusterIssuer + metadata: + name: letsencrypt-prod + spec: + acme: + email: ${var.cert_manager_acme_email} + server: https://acme-v02.api.letsencrypt.org/directory + privateKeySecretRef: + name: letsencrypt-prod-key + solvers: + - http01: + ingress: {} + EOT + ) +} diff --git a/terraform-env.sh b/terraform-env.sh index a8c7fc0..e301542 100755 --- a/terraform-env.sh +++ b/terraform-env.sh @@ -20,6 +20,11 @@ if [[ -z "${EXTERNAL_DNS_TOKEN+x}" ]]; then exit 1 fi +if [[ -z "${CERT_MANAGER_ACME_EMAIL+x}" ]]; then + echo "Missing required environment variable: CERT_MANAGER_ACME_EMAIL" >&2 + exit 1 +fi + stateBucket="do-foundations-$environment-terraform" stateKey="$service/$environment.tfstate" @@ -32,6 +37,7 @@ tfCliArgsInit=( tfCliArgsPlan=( "-var=environment=$environment" "-var=external_dns_token=$EXTERNAL_DNS_TOKEN" + "-var=cert_manager_acme_email=$CERT_MANAGER_ACME_EMAIL" "-var-file=$environment.tfvars" )