Enabling validation webhook (#170)

* Enabling validation webhook * Fixing mirrord webhook certificate --------- Co-authored-by: Richard Kovacs <[email protected]>
linode · Feb 29, 2024 · 2b7db85 · 2b7db85
1 parent c71af44
commit 2b7db85
Show file tree

Hide file tree

Showing 14 changed files with 218 additions and 30 deletions.
diff --git a/.github/workflows/kuttl-e2e-test-1.26.yaml b/.github/workflows/kuttl-e2e-test-1.26.yaml
@@ -30,6 +30,7 @@ jobs:
       - name: Run kuttl tests 1.26
         run: make kuttl
         env:
+          WEBHOOK_TYPE: 'cert-manager'
           LATEST_KUBE_VERSION: '1.26'
           AWS_ACCESS_KEY_ID: 'Dummy'
           AWS_SECRET_ACCESS_KEY: 'Dummy'

diff --git a/.github/workflows/kuttl-e2e-test-1.27.yaml b/.github/workflows/kuttl-e2e-test-1.27.yaml
@@ -30,6 +30,7 @@ jobs:
       - name: Run kuttl tests 1.27
         run: make kuttl
         env:
+          WEBHOOK_TYPE: 'cert-manager'
           LATEST_KUBE_VERSION: '1.27'
           AWS_ACCESS_KEY_ID: 'Dummy'
           AWS_SECRET_ACCESS_KEY: 'Dummy'

diff --git a/.github/workflows/kuttl-e2e-test-1.28.yaml b/.github/workflows/kuttl-e2e-test-1.28.yaml
@@ -30,6 +30,7 @@ jobs:
       - name: Run kuttl tests 1.28
         run: make kuttl
         env:
+          WEBHOOK_TYPE: 'cert-manager'
           LATEST_KUBE_VERSION: '1.28'
           AWS_ACCESS_KEY_ID: 'Dummy'
           AWS_SECRET_ACCESS_KEY: 'Dummy'

diff --git a/.github/workflows/kuttl-e2e-test-1.29.yaml b/.github/workflows/kuttl-e2e-test-1.29.yaml
@@ -30,6 +30,7 @@ jobs:
       - name: Run kuttl tests 1.29
         run: make kuttl
         env:
+          WEBHOOK_TYPE: 'cert-manager'
           LATEST_KUBE_VERSION: '1.29'
           AWS_ACCESS_KEY_ID: 'Dummy'
           AWS_SECRET_ACCESS_KEY: 'Dummy'

diff --git a/.husky/hooks/pre-push b/.husky/hooks/pre-push
@@ -13,6 +13,9 @@ if git status --short | grep -qv "??"; then
     trap unstash EXIT
 fi
 
+# Avoid commiting other then stock manifests.
+export WEBHOOK_TYPE=stock
+
 make generate generate-pkg generate-tests
 git diff --exit-code --quiet || (git status && exit 1)
 

diff --git a/.mirrord/mirrord.json b/.mirrord/mirrord.json
@@ -14,7 +14,7 @@
             "outgoing": false
         },
         "fs": "local",
-        "env": true
+        "env": false
     },
     "operator": false,
     "pause": true,

diff --git a/.mirrord/mirrord.mk b/.mirrord/mirrord.mk
@@ -2,10 +2,17 @@ MIRRORD_VERSION ?= 3.88.0
 MIRRORD := $(TOOLS_HOST_DIR)/mirrord-$(MIRRORD_VERSION)
 
 # Best for development - locally run provider-ceph controller.
-mirrord.cluster: generate-pkg generate-tests crossplane-cluster localstack-cluster load-package
+mirrord.cluster: generate-pkg generate-tests crossplane-cluster localstack-cluster cert-manager load-package mirrord.certs
 	@$(KUBECTL) apply -R -f package/crds
-	# @$(KUBECTL) apply -R -f package/webhookconfigurations
-	$(KUBECTL) apply -f $(PROJECT_ROOT)/e2e/localstack/localstack-provider-cfg.yaml
+	@$(KUBECTL) apply -R -f package/webhookconfigurations
+	$(KUBECTL) apply -f $(PROJECT_ROOT)/e2e/localstack/localstack-provider-cfg-host.yaml
+
+mirrord.certs: $(KUBECTL)
+	@rm -rf $(PWD)/bin/certs ; mkdir $(PWD)/bin/certs
+	@$(KUBECTL) get secret -n crossplane-system crossplane-provider-provider-ceph-server-cert -o 'go-template={{index .data "ca.crt"}}' | base64 -d >> $(PWD)/bin/certs/ca.crt
+	@$(KUBECTL) get secret -n crossplane-system crossplane-provider-provider-ceph-server-cert -o 'go-template={{index .data "tls.crt"}}' | base64 -d >> $(PWD)/bin/certs/tls.crt
+	@$(KUBECTL) get secret -n crossplane-system crossplane-provider-provider-ceph-server-cert -o 'go-template={{index .data "tls.key"}}' | base64 -d >> $(PWD)/bin/certs/tls.key
+	@chmod 400 $(PWD)/bin/certs/*.crt
 
 mirrord.run: $(MIRRORD)
 	@$(INFO) Starting mirrord on deployment

diff --git a/Makefile b/Makefile
@@ -30,6 +30,8 @@ WEBHOOK_TUNNEL_PORT ?= 9999
 # For local development, subdomain of locatunnel.
 WEBHOOK_SUBDOMAIN ?= $(PROJECT_NAME)-$(shell git rev-parse --short HEAD)-$(shell date +%s)
 
+WEBHOOK_TYPE ?= stock
+
 # ====================================================================================
 # Setup Output
 
@@ -148,6 +150,10 @@ crossplane-cluster: $(HELM3) cluster
 	@$(HELM3) install crossplane --namespace crossplane-system --create-namespace --version $(CROSSPLANE_VERSION) crossplane-stable/crossplane
 	@$(OK) Installing Crossplane
 
+## Deploy cert manager to the K8s cluster specified in ~/.kube/config.
+cert-manager: $(KUBECTL)
+	$(KUBECTL) apply -f https://github.com/cert-manager/cert-manager/releases/download/v1.14.0/cert-manager.yaml
+
 # Generate the provider-ceph package and webhookconfiguration manifest.
 generate-pkg: generate kustomize-webhook
 
@@ -159,7 +165,7 @@ check-diff-pkg: generate-pkg
 # Kustomize the webhookconfiguration manifest that is created by 'generate' target.
 kustomize-webhook: $(KUSTOMIZE)
 	@cp -f $(XPKG_DIR)/webhookconfigurations/manifests.yaml $(VAL_WBHK_STAGE)
-	@cp -f $(VAL_WBHK_STAGE)/service-patch-prod.yaml $(VAL_WBHK_STAGE)/service-patch.yaml
+	@cp -f $(VAL_WBHK_STAGE)/service-patch-$(WEBHOOK_TYPE).yaml $(VAL_WBHK_STAGE)/service-patch.yaml
 	$(KUSTOMIZE) build $(VAL_WBHK_STAGE) -o $(XPKG_DIR)/webhookconfigurations/manifests.yaml
 
 # Build the controller image and the provider package.
@@ -181,7 +187,7 @@ load-package: $(KIND) build kustomize-webhook
 # to the Provider.
 # Run Kuttl test suite on newly built controller image.
 # Destroy Kind and localstack.
-kuttl: $(KUTTL) crossplane-cluster localstack-cluster load-package
+kuttl: $(KUTTL) generate-pkg generate-tests crossplane-cluster localstack-cluster cert-manager load-package
 	@$(INFO) Running kuttl test suite
 	@$(KUTTL) test --config e2e/kuttl/stable/provider-ceph-$(LATEST_KUBE_VERSION).yaml
 	@$(OK) Running kuttl test suite
@@ -197,13 +203,27 @@ ceph-kuttl: $(KIND) $(KUTTL) $(HELM3) cluster-clean
 # containerised Crossplane componenets).
 # Install local provider-ceph CRDs.
 # Create ProviderConfig CR representing localstack.
-dev-cluster: $(KUBECTL) generate-pkg generate-tests crossplane-cluster localstack-cluster
-	@rm -rf $(PWD)/bin/certs ; mkdir $(PWD)/bin/certs
-	@docker cp local-dev-control-plane:/etc/kubernetes/pki/ca.key $(PWD)/bin/certs/ca.key
-	@docker cp local-dev-control-plane:/etc/kubernetes/pki/ca.crt $(PWD)/bin/certs/ca.crt
-	@chmod 400 $(PWD)/bin/certs/ca.crt
+dev-cluster: $(KUBECTL) generate-pkg generate-tests crossplane-cluster localstack-cluster cert-manager
+	@$(INFO) Rendering webhook manifest.
+	@ex $(VAL_WBHK_STAGE)/service-patch-dev.tpl.yaml \
+		-c '%s/#WEBHOOK_HOST#/$(WEBHOOK_SUBDOMAIN).loca.lt/' \
+		-c 'sav! $(VAL_WBHK_STAGE)/service-patch.yaml' \
+		-c 'q' >/dev/null
+	$(KUSTOMIZE) build $(VAL_WBHK_STAGE) -o $(XPKG_DIR)/webhookconfigurations/manifests.yaml
+	@$(OK) Rendering webhook manifest.
+
+	@$(INFO) Installing CRDs, ProviderConfig and Localstack
+	@$(KUBECTL) apply -R -f package/crds
+	@$(KUBECTL) apply -R -f package/webhookconfigurations
+	@$(KUBECTL) apply -R -f e2e/localstack/localstack-provider-cfg-host.yaml
+	@$(OK) Installing CRDs and ProviderConfig
 
+	@$(INFO) Starting webhook tunnel.
+	@WEBHOOK_TUNNEL_PORT=$(WEBHOOK_TUNNEL_PORT) WEBHOOK_HOST=$(WEBHOOK_HOST) WEBHOOK_SUBDOMAIN=$(WEBHOOK_SUBDOMAIN) docker compose -f ./hack/localtunnel.yaml up -d
+	@$(OK) Starting webhook tunnel.
+
 	@$(INFO) Generate TLS certificate for webhook.
+	@$(MAKE) dev-cluster-cacert
 	@openssl genrsa \
 		-out $(PWD)/bin/certs/tls.key \
 		1024
@@ -223,23 +243,11 @@ dev-cluster: $(KUBECTL) generate-pkg generate-tests crossplane-cluster localstac
 		-CAkey $(PWD)/bin/certs/ca.key
 	@$(OK) Generate TLS certificate for webhook.
 
-	@$(INFO) Rendering webhook manifest.
-	@ex $(VAL_WBHK_STAGE)/service-patch-dev.tpl.yaml \
-		-c '%s/#WEBHOOK_HOST#/$(WEBHOOK_SUBDOMAIN).loca.lt/' \
-		-c 'sav! $(VAL_WBHK_STAGE)/service-patch.yaml' \
-		-c 'q' >/dev/null
-	$(KUSTOMIZE) build $(VAL_WBHK_STAGE) -o $(XPKG_DIR)/webhookconfigurations/manifests.yaml
-	@$(OK) Rendering webhook manifest.
-
-	@$(INFO) Installing CRDs, ProviderConfig and Localstack
-	@$(KUBECTL) apply -R -f package/crds
-	@$(KUBECTL) apply -R -f package/webhookconfigurations
-	@$(KUBECTL) apply -R -f e2e/localstack/localstack-provider-cfg-host.yaml
-	@$(OK) Installing CRDs and ProviderConfig
-
-	@$(INFO) Starting webhook tunnel.
-	@WEBHOOK_TUNNEL_PORT=$(WEBHOOK_TUNNEL_PORT) WEBHOOK_HOST=$(WEBHOOK_HOST) WEBHOOK_SUBDOMAIN=$(WEBHOOK_SUBDOMAIN) docker compose -f ./hack/localtunnel.yaml up -d
-	@$(OK) Starting webhook tunnel.
+dev-cluster-cacert: $(KUBECTL)
+	@rm -rf $(PWD)/bin/certs ; mkdir $(PWD)/bin/certs
+	@docker cp local-dev-control-plane:/etc/kubernetes/pki/ca.key $(PWD)/bin/certs/ca.key
+	@docker cp local-dev-control-plane:/etc/kubernetes/pki/ca.crt $(PWD)/bin/certs/ca.crt
+	@chmod 400 $(PWD)/bin/certs/*.crt
 
 # Best for development - locally run provider-ceph controller.
 # Removes need for Crossplane install via Helm.

diff --git a/README.md b/README.md
@@ -62,5 +62,116 @@ EOF
 ```
 See [WEBHOOKS.md](docs/WEBHOOKS.md) for instructions on how to enable webhooks.
 
+### Customizing provider deployment
+
+Crossplane uses `DeploymentRuntimeConfig` object to apply customizations on the provider.
+Here are a few examples:
+
+
+```yaml
+apiVersion: pkg.crossplane.io/v1beta1
+kind: DeploymentRuntimeConfig
+metadata:
+  name: provider-ceph
+spec:
+  deploymentTemplate:
+    spec:
+      selector: {}
+      template:
+        spec:
+          containers:
+          - name: package-runtime
+            args:
+            - --kube-client-rate=80000
+            - --reconcile-timeout=5s
+            - --max-reconcile-rate=600
+            - --reconcile-concurrency=160
+            - --poll=30m
+            - --sync=1h
+            - --assume-role-arn=[ASSUME_ROLE_ARN]
+```
+
+You have to attach `DeploymentRuntimeConfig` to the `Provider` object.
+
+```yaml
+apiVersion: pkg.crossplane.io/v1
+kind: Provider
+metadata:
+  name: provider-ceph
+spec:
+  runtimeConfigRef:
+    name: provider-ceph
+```
+
+### Enabling validation webhook
+
+Validation webhook is not registered on the Kubernetes cluster by default. But before you enable it,
+you have to deploy cert manager.
+
+```bash
+kubectl apply -f https://github.com/cert-manager/cert-manager/releases/download/v1.14.0/cert-manager.yaml
+```
+
+You have to create Issuer and Certificate.
+
+```yaml
+apiVersion: cert-manager.io/v1
+kind: Issuer
+metadata:
+  name: selfsigned-issuer
+  namespace: crossplane-system
+spec:
+  selfSigned: {}
+---
+apiVersion: cert-manager.io/v1
+kind: Certificate
+metadata:
+  name: crossplane-provider-provider-ceph
+  namespace: crossplane-system
+spec:
+  commonName: provider-ceph.crossplane-system.svc
+  dnsNames:
+  - provider-ceph.crossplane-system.svc.cluster.local
+  - provider-ceph.crossplane-system.svc
+  - crossplane-provider-provider-ceph.crossplane-system.svc.cluster.local
+  - crossplane-provider-provider-ceph.crossplane-system.svc
+  issuerRef:
+    kind: Issuer
+    name: selfsigned-issuer
+  secretName: crossplane-provider-provider-ceph-server-cert
+```
+
+You need a `DeploymentRuntimeConfig` too ([Customizing provider deployment](#customizing-provider-deployment)).
+
+```yaml
+apiVersion: pkg.crossplane.io/v1beta1
+kind: DeploymentRuntimeConfig
+metadata:
+  name: provider-ceph
+spec:
+  deploymentTemplate:
+    spec:
+      selector: {}
+      template:
+        spec:
+          containers:
+          - name: package-runtime
+            args:
+            - --webhook-tls-cert-dir=/certs
+            volumeMounts:
+            - name: cert-manager-certs
+              mountPath: /certs
+          volumes:
+          - name: cert-manager-certs
+            secret:
+              secretName: crossplane-provider-provider-ceph-server-cert
+```
+
+Finaly, you have to apply `ValidatingWebhookConfiguration`.
+
+```bash
+PROV_CEPH_VER=v0.0.32 kubectl apply -f https://github.com/linode/provider-ceph/blob/release-${PROV_CEPH_VER}/package/webhookconfigurations/manifests.yaml
+```
+
 ## Contact
 - Slack: Join our [#provider-ceph](https://crossplane.slack.com/archives/C05RKQRNDHA) slack channel.
diff --git a/e2e/tests/stable/provider-ceph/00-enable-webhook.yaml b/e2e/tests/stable/provider-ceph/00-enable-webhook.yaml
@@ -0,0 +1,4 @@
+apiVersion: kuttl.dev/v1beta1
+kind: TestStep
+commands:
+  - command: kubectl apply -R -f ../../../../package/webhookconfigurations
diff --git a/hack/deploy-provider.sh b/hack/deploy-provider.sh
@@ -5,12 +5,40 @@
 : "${ARCH?= required}"
 : "${VERSION?= required}"
 
+# Apply cert-manager Issuer and Certificate
+kubectl apply -f - <<EOF
+apiVersion: cert-manager.io/v1
+kind: Issuer
+metadata:
+  name: selfsigned-issuer
+  namespace: crossplane-system
+spec:
+  selfSigned: {}
+---
+apiVersion: cert-manager.io/v1
+kind: Certificate
+metadata:
+  name: crossplane-provider-provider-ceph
+  namespace: crossplane-system
+spec:
+  commonName: provider-ceph.crossplane-system.svc
+  dnsNames:
+  - provider-ceph.crossplane-system.svc.cluster.local
+  - provider-ceph.crossplane-system.svc
+  - crossplane-provider-provider-ceph.crossplane-system.svc.cluster.local
+  - crossplane-provider-provider-ceph.crossplane-system.svc
+  issuerRef:
+    kind: Issuer
+    name: selfsigned-issuer
+  secretName: crossplane-provider-provider-ceph-server-cert
+EOF
+
 # Apply a configuration for the provider deployment.
 kubectl apply -f - <<EOF
 apiVersion: pkg.crossplane.io/v1beta1
 kind: DeploymentRuntimeConfig
 metadata:
-  name: config
+  name: provider-ceph
 spec:
   deploymentTemplate:
     spec:
@@ -29,6 +57,14 @@ spec:
             - --poll=30m
             - --sync=1h
             - --assume-role-arn=${ASSUME_ROLE_ARN}
+            - --webhook-tls-cert-dir=/certs
+            volumeMounts:
+            - name: cert-manager-certs
+              mountPath: /certs
+          volumes:
+          - name: cert-manager-certs
+            secret:
+              secretName: crossplane-provider-provider-ceph-server-cert
 EOF
 
 # Apply the provider.
@@ -41,6 +77,6 @@ spec:
   package: ${PROJECT_NAME}-${VERSION}.gz
   packagePullPolicy: Never
   runtimeConfigRef:
-    name: config
+    name: provider-ceph
 EOF
 
diff --git a/hack/generate-tests.sh b/hack/generate-tests.sh
@@ -120,6 +120,7 @@ jobs:
       - name: Run kuttl tests ${major}
         run: make kuttl
         env:
+          WEBHOOK_TYPE: 'cert-manager'
           LATEST_KUBE_VERSION: '${major}'
           AWS_ACCESS_KEY_ID: 'Dummy'
           AWS_SECRET_ACCESS_KEY: 'Dummy'

diff --git a/staging/validatingwebhookconfiguration/service-patch-cert-manager.yaml b/staging/validatingwebhookconfiguration/service-patch-cert-manager.yaml
@@ -0,0 +1,14 @@
+apiVersion: admissionregistration.k8s.io/v1
+kind: ValidatingWebhookConfiguration
+metadata:
+  name: validating-webhook-configuration
+  annotations:
+    cert-manager.io/inject-ca-from: crossplane-system/crossplane-provider-provider-ceph
+webhooks:
+- name: bucket-validation.providerceph.crossplane.io
+  clientConfig:
+    caBundle: Cg==
+    service:
+      name: provider-ceph
+      namespace: crossplane-system
+      port: 9443
diff --git a/...hookconfiguration/service-patch-prod.yaml → ...ookconfiguration/service-patch-stock.yaml b/...hookconfiguration/service-patch-prod.yaml → ...ookconfiguration/service-patch-stock.yaml