Add initial Cilium support

modules/azure/aks/aks.tf

@@ -34,8 +34,7 @@ resource "azurerm_kubernetes_cluster" "this" {
   }
 
   network_profile {
-    network_plugin    = "kubenet"
-    network_policy    = "calico"
+    network_plugin    = "none"
     load_balancer_sku = "standard"
     load_balancer_profile {
       outbound_ip_prefix_ids = [

modules/kubernetes/README.md

@@ -9,6 +9,7 @@ This directory contains all the Kubernetes Terraform modules.
 - [`aad-pod-identity`](aad-pod-identity/README.md)
 - [`azure-metrics`](azure-metrics/README.md)
 - [`cert-manager`](cert-manager/README.md)
+- [`cilium`](cilium/README.md)
 - [`csi-secrets-store-provider-azure`](csi-secrets-store-provider-azure/README.md)
 - [`csi-secrets-store-provider-aws`](csi-secrets-store-provider-aws/README.md)
 - [`external-dns`](external-dns/README.md)

modules/kubernetes/aks-core/README.md

@@ -34,6 +34,7 @@ This module is used to create AKS clusters.
 | <a name="module_azure_metrics"></a> [azure\_metrics](#module\_azure\_metrics) | ../../kubernetes/azure-metrics | n/a |
 | <a name="module_cert_manager"></a> [cert\_manager](#module\_cert\_manager) | ../../kubernetes/cert-manager | n/a |
 | <a name="module_cert_manager_crd"></a> [cert\_manager\_crd](#module\_cert\_manager\_crd) | ../../kubernetes/helm-crd | n/a |
+| <a name="module_cilium"></a> [cilium](#module\_cilium) | ../../kubernetes/cilium | n/a |
 | <a name="module_control_plane_logs"></a> [control\_plane\_logs](#module\_control\_plane\_logs) | ../../kubernetes/control-plane-logs | n/a |
 | <a name="module_csi_secrets_store_provider_azure"></a> [csi\_secrets\_store\_provider\_azure](#module\_csi\_secrets\_store\_provider\_azure) | ../../kubernetes/csi-secrets-store-provider-azure | n/a |
 | <a name="module_csi_secrets_store_provider_azure_crd"></a> [csi\_secrets\_store\_provider\_azure\_crd](#module\_csi\_secrets\_store\_provider\_azure\_crd) | ../../kubernetes/helm-crd | n/a |
@@ -112,6 +113,7 @@ This module is used to create AKS clusters.
 | <a name="input_azure_metrics_enabled"></a> [azure\_metrics\_enabled](#input\_azure\_metrics\_enabled) | Should AZ Metrics be enabled | `bool` | `true` | no |
 | <a name="input_cert_manager_config"></a> [cert\_manager\_config](#input\_cert\_manager\_config) | Cert Manager configuration, the first item in the list is the main domain | <pre>object({<br>    notification_email = string<br>    dns_zone           = list(string)<br>  })</pre> | n/a | yes |
 | <a name="input_cert_manager_enabled"></a> [cert\_manager\_enabled](#input\_cert\_manager\_enabled) | Should Cert Manager be enabled | `bool` | `true` | no |
+| <a name="input_cilium_enabled"></a> [cilium\_enabled](#input\_cilium\_enabled) | Should Cilium be enabled | `bool` | `false` | no |
 | <a name="input_control_plane_logs_config"></a> [control\_plane\_logs\_config](#input\_control\_plane\_logs\_config) | Configuration for control plane log | <pre>object({<br>    azure_key_vault_name = string<br>    identity = object({<br>      client_id   = string<br>      resource_id = string<br>      tenant_id   = string<br>    })<br>    eventhub_hostname = string<br>    eventhub_name     = string<br>  })</pre> | <pre>{<br>  "azure_key_vault_name": "",<br>  "eventhub_hostname": "",<br>  "eventhub_name": "",<br>  "identity": {<br>    "client_id": "",<br>    "resource_id": "",<br>    "tenant_id": ""<br>  }<br>}</pre> | no |
 | <a name="input_control_plane_logs_enabled"></a> [control\_plane\_logs\_enabled](#input\_control\_plane\_logs\_enabled) | Should Control plan be enabled | `bool` | `false` | no |
 | <a name="input_csi_secrets_store_provider_azure_enabled"></a> [csi\_secrets\_store\_provider\_azure\_enabled](#input\_csi\_secrets\_store\_provider\_azure\_enabled) | Should csi-secrets-store-provider-azure be enabled | `bool` | `true` | no |

modules/kubernetes/aks-core/modules.tf

@@ -24,6 +24,17 @@ locals {
   ]
 }
 
+module "cilium" {
+  for_each = {
+    for s in ["cilium"] :
+    s => s
+    if var.cilium_enabled
+  }
+
+  source = "../../kubernetes/cilium"
+}
+
+
 # OPA Gatekeeper
 module "opa_gatekeeper_crd" {
   source = "../../kubernetes/helm-crd"
@@ -34,7 +45,7 @@ module "opa_gatekeeper_crd" {
 }
 
 module "opa_gatekeeper" {
-  depends_on = [module.opa_gatekeeper_crd]
+  depends_on = [module.opa_gatekeeper_crd, module.cilium]
 
   for_each = {
     for s in ["opa-gatekeeper"] :
@@ -79,6 +90,7 @@ module "opa_gatekeeper" {
 
 # FluxCD v2
 module "fluxcd_v2_azure_devops" {
+  depends_on = [module.cilium]
   for_each = {
     for s in ["fluxcd-v2"] :
     s => s
@@ -105,6 +117,7 @@ module "fluxcd_v2_azure_devops" {
 }
 
 module "fluxcd_v2_github" {
+  depends_on = [module.cilium]
   for_each = {
     for s in ["fluxcd-v2"] :
     s => s
@@ -187,7 +200,7 @@ module "linkerd_crd" {
 }
 
 module "linkerd" {
-  depends_on = [module.opa_gatekeeper, module.cert_manager_crd, module.linkerd_crd]
+  depends_on = [module.opa_gatekeeper, module.cert_manager_crd, module.linkerd_crd, module.cert_manager]
 
   for_each = {
     for s in ["linkerd"] :
@@ -524,6 +537,7 @@ module "prometheus" {
   node_local_dns_enabled                   = var.node_local_dns_enabled
   grafana_agent_enabled                    = var.grafana_agent_enabled
   promtail_enabled                         = var.promtail_enabled
+  cilium_enabled                           = var.cilium_enabled
 }
 
 module "control_plane_logs" {

modules/kubernetes/aks-core/variables.tf

@@ -522,3 +522,8 @@ variable "control_plane_logs_config" {
     eventhub_name     = ""
   }
 }
+variable "cilium_enabled" {
+  description = "Should Cilium be enabled"
+  type        = bool
+  default     = false
+}

modules/kubernetes/cilium/README.md

@@ -0,0 +1,31 @@
+## Requirements
+
+| Name | Version |
+|------|---------|
+| <a name="requirement_terraform"></a> [terraform](#requirement\_terraform) | >= 1.3.0 |
+| <a name="requirement_helm"></a> [helm](#requirement\_helm) | 2.6.0 |
+| <a name="requirement_kubernetes"></a> [kubernetes](#requirement\_kubernetes) | 2.13.1 |
+
+## Providers
+
+| Name | Version |
+|------|---------|
+| <a name="provider_helm"></a> [helm](#provider\_helm) | 2.6.0 |
+
+## Modules
+
+No modules.
+
+## Resources
+
+| Name | Type |
+|------|------|
+| [helm_release.this](https://registry.terraform.io/providers/hashicorp/helm/2.6.0/docs/resources/release) | resource |
+
+## Inputs
+
+No inputs.
+
+## Outputs
+
+No outputs.

modules/kubernetes/cilium/main.tf

@@ -0,0 +1,27 @@
+terraform {
+  required_version = ">= 1.3.0"
+
+  required_providers {
+    kubernetes = {
+      source  = "hashicorp/kubernetes"
+      version = "2.13.1"
+    }
+    helm = {
+      source  = "hashicorp/helm"
+      version = "2.6.0"
+    }
+  }
+}
+
+resource "helm_release" "this" {
+  repository  = "https://helm.cilium.io/"
+  chart       = "cilium"
+  name        = "cilium"
+  namespace   = "kube-system"
+  version     = "1.12.2"
+  max_history = 3
+
+  values = [
+    templatefile("${path.module}/templates/values.yaml.tpl", {}),
+  ]
+}

modules/kubernetes/cilium/templates/values.yaml.tpl

@@ -0,0 +1,58 @@
+aksbyocni:
+  enabled: true
+
+prometheus:
+  enabled: true
+  serviceMonitor:
+    enabled: true
+    labels:
+      xkf.xenit.io/monitoring: platform
+
+operator:
+  prometheus:
+    enabled: true
+    serviceMonitor:
+      enabled: true
+      labels:
+        xkf.xenit.io/monitoring: platform
+
+hubble:
+  enabled: true
+  metrics:
+    enabled:
+      - dns:query;ignoreAAAA
+      - drop
+      - tcp
+      - flow
+      - icmp
+      - http
+    serviceMonitor:
+      enabled: true
+      labels:
+        xkf.xenit.io/monitoring: platform
+
+  relay:
+    enabled: true
+    prometheus:
+      enabled: true
+      serviceMonitor:
+        enabled: true
+        labels:
+          xkf.xenit.io/monitoring: platform
+
+  ui:
+    enabled: true
+
+hostPort:
+  enabled: true
+
+nodePort:
+  enabled: true
+
+bpf:
+  masquerade: true
+
+kubeProxyReplacement: partial
+
+externalIPs:
+  enabled: true

modules/kubernetes/linkerd/templates/values-cni.yaml.tpl

@@ -13,7 +13,7 @@ extraInitContainers:
       - -xc
       - |
         for i in $(seq 1 180); do
-          test -f /host/etc/cni/net.d/10-calico.conflist && exit 0
+          test -f /host/etc/cni/net.d/05-cilium.conf && exit 0
           sleep 1
         done
         exit 1

modules/kubernetes/prometheus/README.md

@@ -39,6 +39,7 @@ No modules.
 | <a name="input_aws_config"></a> [aws\_config](#input\_aws\_config) | AWS specific configuration | <pre>object({<br>    role_arn = string<br>  })</pre> | <pre>{<br>  "role_arn": ""<br>}</pre> | no |
 | <a name="input_azad_kube_proxy_enabled"></a> [azad\_kube\_proxy\_enabled](#input\_azad\_kube\_proxy\_enabled) | Should azad-kube-proxy be enabled | `bool` | `false` | no |
 | <a name="input_azure_config"></a> [azure\_config](#input\_azure\_config) | Azure specific configuration | <pre>object({<br>    azure_key_vault_name = string<br>    identity = object({<br>      client_id   = string<br>      resource_id = string<br>      tenant_id   = string<br>    })<br>  })</pre> | <pre>{<br>  "azure_key_vault_name": "",<br>  "identity": {<br>    "client_id": "",<br>    "resource_id": "",<br>    "tenant_id": ""<br>  }<br>}</pre> | no |
+| <a name="input_cilium_enabled"></a> [cilium\_enabled](#input\_cilium\_enabled) | Should promtail be enabled | `bool` | `false` | no |
 | <a name="input_cloud_provider"></a> [cloud\_provider](#input\_cloud\_provider) | Name of cloud provider | `string` | n/a | yes |
 | <a name="input_cluster_name"></a> [cluster\_name](#input\_cluster\_name) | Name of the prometheus cluster | `string` | n/a | yes |
 | <a name="input_csi_secrets_store_provider_aws_enabled"></a> [csi\_secrets\_store\_provider\_aws\_enabled](#input\_csi\_secrets\_store\_provider\_aws\_enabled) | Should csi-secrets-store-provider-aws be enabled | `bool` | `false` | no |

modules/kubernetes/prometheus/charts/prometheus-extras/templates/monitors.yaml

@@ -445,4 +445,59 @@ spec:
       app.kubernetes.io/instance: promtail
       app.kubernetes.io/name: promtail
 {{- end }}
-
+{{- if .Values.enabledMonitors.cilium }}
+---
+apiVersion: monitoring.coreos.com/v1
+kind: PodMonitor
+metadata:
+  name: cilium-agent
+  namespace: prometheus
+  labels:
+    xkf.xenit.io/monitoring: platform
+spec:
+  selector:
+    matchLabels:
+      k8s-app: cilium
+  podMetricsEndpoints:
+    - path: /metrics
+      port: prometheus
+  namespaceSelector:
+    matchNames:
+      - kube-system
+---
+apiVersion: monitoring.coreos.com/v1
+kind: PodMonitor
+metadata:
+  name: cilium-operator
+  namespace: prometheus
+  labels:
+    xkf.xenit.io/monitoring: platform
+spec:
+  selector:
+    matchLabels:
+      name: cilium-operator
+  podMetricsEndpoints:
+    - path: /metrics
+      port: prometheus
+  namespaceSelector:
+    matchNames:
+      - kube-system
+---
+apiVersion: monitoring.coreos.com/v1
+kind: PodMonitor
+metadata:
+  name: hubble-relay
+  namespace: prometheus
+  labels:
+    xkf.xenit.io/monitoring: platform
+spec:
+  selector:
+    matchLabels:
+      k8s-app: hubble-relay
+  podMetricsEndpoints:
+    - path: /metrics
+      port: prometheus
+  namespaceSelector:
+    matchNames:
+      - kube-system
+{{- end }}

modules/kubernetes/prometheus/charts/prometheus-extras/values.yaml

@@ -68,3 +68,4 @@ enabledMonitors:
   grafanaAgent: false
   nodeLocalDNS: false
   promtail: false
+  cilium: false

modules/kubernetes/prometheus/main.tf

@@ -100,5 +100,6 @@ resource "helm_release" "prometheus_extras" {
     grafana_agent_enabled                    = var.grafana_agent_enabled
     node_local_dns_enabled                   = var.node_local_dns_enabled
     promtail_enabled                         = var.promtail_enabled
+    cilium_enabled                           = var.cilium_enabled
   })]
 }

modules/kubernetes/prometheus/templates/values-extras.yaml.tpl

@@ -49,3 +49,4 @@ enabledMonitors:
   grafanaAgent: ${grafana_agent_enabled}
   nodeLocalDNS: ${node_local_dns_enabled}
   promtail: ${promtail_enabled}
+  cilium: ${cilium_enabled}

modules/kubernetes/prometheus/variables.tf

@@ -168,3 +168,9 @@ variable "promtail_enabled" {
   type        = bool
   default     = false
 }
+
+variable "cilium_enabled" {
+  description = "Should promtail be enabled"
+  type        = bool
+  default     = false
+}

validation/kubernetes/cilium/main.tf

@@ -0,0 +1,15 @@
+terraform {}
+
+provider "kubernetes" {}
+
+provider "helm" {}
+
+module "cilium" {
+  source = "../../../modules/kubernetes/cilium"
+
+  providers = {
+    kubernetes = kubernetes
+    helm       = helm
+  }
+
+}