From 2b7db850dd5acc5419cf29e1111ff305d3d8db15 Mon Sep 17 00:00:00 2001 From: Richard Kovacs Date: Thu, 29 Feb 2024 17:15:29 +0100 Subject: [PATCH] Enabling validation webhook (#170) * Enabling validation webhook * Fixing mirrord webhook certificate --------- Co-authored-by: Richard Kovacs --- .github/workflows/kuttl-e2e-test-1.26.yaml | 1 + .github/workflows/kuttl-e2e-test-1.27.yaml | 1 + .github/workflows/kuttl-e2e-test-1.28.yaml | 1 + .github/workflows/kuttl-e2e-test-1.29.yaml | 1 + .husky/hooks/pre-push | 3 + .mirrord/mirrord.json | 2 +- .mirrord/mirrord.mk | 13 +- Makefile | 56 +++++---- README.md | 111 ++++++++++++++++++ .../provider-ceph/00-enable-webhook.yaml | 4 + hack/deploy-provider.sh | 40 ++++++- hack/generate-tests.sh | 1 + .../service-patch-cert-manager.yaml | 14 +++ ...tch-prod.yaml => service-patch-stock.yaml} | 0 14 files changed, 218 insertions(+), 30 deletions(-) create mode 100644 e2e/tests/stable/provider-ceph/00-enable-webhook.yaml create mode 100644 staging/validatingwebhookconfiguration/service-patch-cert-manager.yaml rename staging/validatingwebhookconfiguration/{service-patch-prod.yaml => service-patch-stock.yaml} (100%) diff --git a/.github/workflows/kuttl-e2e-test-1.26.yaml b/.github/workflows/kuttl-e2e-test-1.26.yaml index c7f61e1a..91671f48 100644 --- a/.github/workflows/kuttl-e2e-test-1.26.yaml +++ b/.github/workflows/kuttl-e2e-test-1.26.yaml @@ -30,6 +30,7 @@ jobs: - name: Run kuttl tests 1.26 run: make kuttl env: + WEBHOOK_TYPE: 'cert-manager' LATEST_KUBE_VERSION: '1.26' AWS_ACCESS_KEY_ID: 'Dummy' AWS_SECRET_ACCESS_KEY: 'Dummy' diff --git a/.github/workflows/kuttl-e2e-test-1.27.yaml b/.github/workflows/kuttl-e2e-test-1.27.yaml index a77ead1e..7077c173 100644 --- a/.github/workflows/kuttl-e2e-test-1.27.yaml +++ b/.github/workflows/kuttl-e2e-test-1.27.yaml @@ -30,6 +30,7 @@ jobs: - name: Run kuttl tests 1.27 run: make kuttl env: + WEBHOOK_TYPE: 'cert-manager' LATEST_KUBE_VERSION: '1.27' AWS_ACCESS_KEY_ID: 'Dummy' AWS_SECRET_ACCESS_KEY: 'Dummy' diff --git a/.github/workflows/kuttl-e2e-test-1.28.yaml b/.github/workflows/kuttl-e2e-test-1.28.yaml index bcb3968f..37eef808 100644 --- a/.github/workflows/kuttl-e2e-test-1.28.yaml +++ b/.github/workflows/kuttl-e2e-test-1.28.yaml @@ -30,6 +30,7 @@ jobs: - name: Run kuttl tests 1.28 run: make kuttl env: + WEBHOOK_TYPE: 'cert-manager' LATEST_KUBE_VERSION: '1.28' AWS_ACCESS_KEY_ID: 'Dummy' AWS_SECRET_ACCESS_KEY: 'Dummy' diff --git a/.github/workflows/kuttl-e2e-test-1.29.yaml b/.github/workflows/kuttl-e2e-test-1.29.yaml index 71003491..e7cf2926 100644 --- a/.github/workflows/kuttl-e2e-test-1.29.yaml +++ b/.github/workflows/kuttl-e2e-test-1.29.yaml @@ -30,6 +30,7 @@ jobs: - name: Run kuttl tests 1.29 run: make kuttl env: + WEBHOOK_TYPE: 'cert-manager' LATEST_KUBE_VERSION: '1.29' AWS_ACCESS_KEY_ID: 'Dummy' AWS_SECRET_ACCESS_KEY: 'Dummy' diff --git a/.husky/hooks/pre-push b/.husky/hooks/pre-push index 3d88dc2d..1c3378e9 100755 --- a/.husky/hooks/pre-push +++ b/.husky/hooks/pre-push @@ -13,6 +13,9 @@ if git status --short | grep -qv "??"; then trap unstash EXIT fi +# Avoid commiting other then stock manifests. +export WEBHOOK_TYPE=stock + make generate generate-pkg generate-tests git diff --exit-code --quiet || (git status && exit 1) diff --git a/.mirrord/mirrord.json b/.mirrord/mirrord.json index ad5328d2..63ff3ea3 100644 --- a/.mirrord/mirrord.json +++ b/.mirrord/mirrord.json @@ -14,7 +14,7 @@ "outgoing": false }, "fs": "local", - "env": true + "env": false }, "operator": false, "pause": true, diff --git a/.mirrord/mirrord.mk b/.mirrord/mirrord.mk index 244d4754..8ac30550 100644 --- a/.mirrord/mirrord.mk +++ b/.mirrord/mirrord.mk @@ -2,10 +2,17 @@ MIRRORD_VERSION ?= 3.88.0 MIRRORD := $(TOOLS_HOST_DIR)/mirrord-$(MIRRORD_VERSION) # Best for development - locally run provider-ceph controller. -mirrord.cluster: generate-pkg generate-tests crossplane-cluster localstack-cluster load-package +mirrord.cluster: generate-pkg generate-tests crossplane-cluster localstack-cluster cert-manager load-package mirrord.certs @$(KUBECTL) apply -R -f package/crds - # @$(KUBECTL) apply -R -f package/webhookconfigurations - $(KUBECTL) apply -f $(PROJECT_ROOT)/e2e/localstack/localstack-provider-cfg.yaml + @$(KUBECTL) apply -R -f package/webhookconfigurations + $(KUBECTL) apply -f $(PROJECT_ROOT)/e2e/localstack/localstack-provider-cfg-host.yaml + +mirrord.certs: $(KUBECTL) + @rm -rf $(PWD)/bin/certs ; mkdir $(PWD)/bin/certs + @$(KUBECTL) get secret -n crossplane-system crossplane-provider-provider-ceph-server-cert -o 'go-template={{index .data "ca.crt"}}' | base64 -d >> $(PWD)/bin/certs/ca.crt + @$(KUBECTL) get secret -n crossplane-system crossplane-provider-provider-ceph-server-cert -o 'go-template={{index .data "tls.crt"}}' | base64 -d >> $(PWD)/bin/certs/tls.crt + @$(KUBECTL) get secret -n crossplane-system crossplane-provider-provider-ceph-server-cert -o 'go-template={{index .data "tls.key"}}' | base64 -d >> $(PWD)/bin/certs/tls.key + @chmod 400 $(PWD)/bin/certs/*.crt mirrord.run: $(MIRRORD) @$(INFO) Starting mirrord on deployment diff --git a/Makefile b/Makefile index fcce0844..602325e9 100644 --- a/Makefile +++ b/Makefile @@ -30,6 +30,8 @@ WEBHOOK_TUNNEL_PORT ?= 9999 # For local development, subdomain of locatunnel. WEBHOOK_SUBDOMAIN ?= $(PROJECT_NAME)-$(shell git rev-parse --short HEAD)-$(shell date +%s) +WEBHOOK_TYPE ?= stock + # ==================================================================================== # Setup Output @@ -148,6 +150,10 @@ crossplane-cluster: $(HELM3) cluster @$(HELM3) install crossplane --namespace crossplane-system --create-namespace --version $(CROSSPLANE_VERSION) crossplane-stable/crossplane @$(OK) Installing Crossplane +## Deploy cert manager to the K8s cluster specified in ~/.kube/config. +cert-manager: $(KUBECTL) + $(KUBECTL) apply -f https://github.com/cert-manager/cert-manager/releases/download/v1.14.0/cert-manager.yaml + # Generate the provider-ceph package and webhookconfiguration manifest. generate-pkg: generate kustomize-webhook @@ -159,7 +165,7 @@ check-diff-pkg: generate-pkg # Kustomize the webhookconfiguration manifest that is created by 'generate' target. kustomize-webhook: $(KUSTOMIZE) @cp -f $(XPKG_DIR)/webhookconfigurations/manifests.yaml $(VAL_WBHK_STAGE) - @cp -f $(VAL_WBHK_STAGE)/service-patch-prod.yaml $(VAL_WBHK_STAGE)/service-patch.yaml + @cp -f $(VAL_WBHK_STAGE)/service-patch-$(WEBHOOK_TYPE).yaml $(VAL_WBHK_STAGE)/service-patch.yaml $(KUSTOMIZE) build $(VAL_WBHK_STAGE) -o $(XPKG_DIR)/webhookconfigurations/manifests.yaml # Build the controller image and the provider package. @@ -181,7 +187,7 @@ load-package: $(KIND) build kustomize-webhook # to the Provider. # Run Kuttl test suite on newly built controller image. # Destroy Kind and localstack. -kuttl: $(KUTTL) crossplane-cluster localstack-cluster load-package +kuttl: $(KUTTL) generate-pkg generate-tests crossplane-cluster localstack-cluster cert-manager load-package @$(INFO) Running kuttl test suite @$(KUTTL) test --config e2e/kuttl/stable/provider-ceph-$(LATEST_KUBE_VERSION).yaml @$(OK) Running kuttl test suite @@ -197,13 +203,27 @@ ceph-kuttl: $(KIND) $(KUTTL) $(HELM3) cluster-clean # containerised Crossplane componenets). # Install local provider-ceph CRDs. # Create ProviderConfig CR representing localstack. -dev-cluster: $(KUBECTL) generate-pkg generate-tests crossplane-cluster localstack-cluster - @rm -rf $(PWD)/bin/certs ; mkdir $(PWD)/bin/certs - @docker cp local-dev-control-plane:/etc/kubernetes/pki/ca.key $(PWD)/bin/certs/ca.key - @docker cp local-dev-control-plane:/etc/kubernetes/pki/ca.crt $(PWD)/bin/certs/ca.crt - @chmod 400 $(PWD)/bin/certs/ca.crt +dev-cluster: $(KUBECTL) generate-pkg generate-tests crossplane-cluster localstack-cluster cert-manager + @$(INFO) Rendering webhook manifest. + @ex $(VAL_WBHK_STAGE)/service-patch-dev.tpl.yaml \ + -c '%s/#WEBHOOK_HOST#/$(WEBHOOK_SUBDOMAIN).loca.lt/' \ + -c 'sav! $(VAL_WBHK_STAGE)/service-patch.yaml' \ + -c 'q' >/dev/null + $(KUSTOMIZE) build $(VAL_WBHK_STAGE) -o $(XPKG_DIR)/webhookconfigurations/manifests.yaml + @$(OK) Rendering webhook manifest. + + @$(INFO) Installing CRDs, ProviderConfig and Localstack + @$(KUBECTL) apply -R -f package/crds + @$(KUBECTL) apply -R -f package/webhookconfigurations + @$(KUBECTL) apply -R -f e2e/localstack/localstack-provider-cfg-host.yaml + @$(OK) Installing CRDs and ProviderConfig + @$(INFO) Starting webhook tunnel. + @WEBHOOK_TUNNEL_PORT=$(WEBHOOK_TUNNEL_PORT) WEBHOOK_HOST=$(WEBHOOK_HOST) WEBHOOK_SUBDOMAIN=$(WEBHOOK_SUBDOMAIN) docker compose -f ./hack/localtunnel.yaml up -d + @$(OK) Starting webhook tunnel. + @$(INFO) Generate TLS certificate for webhook. + @$(MAKE) dev-cluster-cacert @openssl genrsa \ -out $(PWD)/bin/certs/tls.key \ 1024 @@ -223,23 +243,11 @@ dev-cluster: $(KUBECTL) generate-pkg generate-tests crossplane-cluster localstac -CAkey $(PWD)/bin/certs/ca.key @$(OK) Generate TLS certificate for webhook. - @$(INFO) Rendering webhook manifest. - @ex $(VAL_WBHK_STAGE)/service-patch-dev.tpl.yaml \ - -c '%s/#WEBHOOK_HOST#/$(WEBHOOK_SUBDOMAIN).loca.lt/' \ - -c 'sav! $(VAL_WBHK_STAGE)/service-patch.yaml' \ - -c 'q' >/dev/null - $(KUSTOMIZE) build $(VAL_WBHK_STAGE) -o $(XPKG_DIR)/webhookconfigurations/manifests.yaml - @$(OK) Rendering webhook manifest. - - @$(INFO) Installing CRDs, ProviderConfig and Localstack - @$(KUBECTL) apply -R -f package/crds - @$(KUBECTL) apply -R -f package/webhookconfigurations - @$(KUBECTL) apply -R -f e2e/localstack/localstack-provider-cfg-host.yaml - @$(OK) Installing CRDs and ProviderConfig - - @$(INFO) Starting webhook tunnel. - @WEBHOOK_TUNNEL_PORT=$(WEBHOOK_TUNNEL_PORT) WEBHOOK_HOST=$(WEBHOOK_HOST) WEBHOOK_SUBDOMAIN=$(WEBHOOK_SUBDOMAIN) docker compose -f ./hack/localtunnel.yaml up -d - @$(OK) Starting webhook tunnel. +dev-cluster-cacert: $(KUBECTL) + @rm -rf $(PWD)/bin/certs ; mkdir $(PWD)/bin/certs + @docker cp local-dev-control-plane:/etc/kubernetes/pki/ca.key $(PWD)/bin/certs/ca.key + @docker cp local-dev-control-plane:/etc/kubernetes/pki/ca.crt $(PWD)/bin/certs/ca.crt + @chmod 400 $(PWD)/bin/certs/*.crt # Best for development - locally run provider-ceph controller. # Removes need for Crossplane install via Helm. diff --git a/README.md b/README.md index c96352f1..20cb3bae 100644 --- a/README.md +++ b/README.md @@ -62,5 +62,116 @@ EOF ``` See [WEBHOOKS.md](docs/WEBHOOKS.md) for instructions on how to enable webhooks. +### Customizing provider deployment + +Crossplane uses `DeploymentRuntimeConfig` object to apply customizations on the provider. +Here are a few examples: + + +```yaml +apiVersion: pkg.crossplane.io/v1beta1 +kind: DeploymentRuntimeConfig +metadata: + name: provider-ceph +spec: + deploymentTemplate: + spec: + selector: {} + template: + spec: + containers: + - name: package-runtime + args: + - --kube-client-rate=80000 + - --reconcile-timeout=5s + - --max-reconcile-rate=600 + - --reconcile-concurrency=160 + - --poll=30m + - --sync=1h + - --assume-role-arn=[ASSUME_ROLE_ARN] +``` + +You have to attach `DeploymentRuntimeConfig` to the `Provider` object. + +```yaml +apiVersion: pkg.crossplane.io/v1 +kind: Provider +metadata: + name: provider-ceph +spec: + runtimeConfigRef: + name: provider-ceph +``` + +### Enabling validation webhook + +Validation webhook is not registered on the Kubernetes cluster by default. But before you enable it, +you have to deploy cert manager. + +```bash +kubectl apply -f https://github.com/cert-manager/cert-manager/releases/download/v1.14.0/cert-manager.yaml +``` + +You have to create Issuer and Certificate. + +```yaml +apiVersion: cert-manager.io/v1 +kind: Issuer +metadata: + name: selfsigned-issuer + namespace: crossplane-system +spec: + selfSigned: {} +--- +apiVersion: cert-manager.io/v1 +kind: Certificate +metadata: + name: crossplane-provider-provider-ceph + namespace: crossplane-system +spec: + commonName: provider-ceph.crossplane-system.svc + dnsNames: + - provider-ceph.crossplane-system.svc.cluster.local + - provider-ceph.crossplane-system.svc + - crossplane-provider-provider-ceph.crossplane-system.svc.cluster.local + - crossplane-provider-provider-ceph.crossplane-system.svc + issuerRef: + kind: Issuer + name: selfsigned-issuer + secretName: crossplane-provider-provider-ceph-server-cert +``` + +You need a `DeploymentRuntimeConfig` too ([Customizing provider deployment](#customizing-provider-deployment)). + +```yaml +apiVersion: pkg.crossplane.io/v1beta1 +kind: DeploymentRuntimeConfig +metadata: + name: provider-ceph +spec: + deploymentTemplate: + spec: + selector: {} + template: + spec: + containers: + - name: package-runtime + args: + - --webhook-tls-cert-dir=/certs + volumeMounts: + - name: cert-manager-certs + mountPath: /certs + volumes: + - name: cert-manager-certs + secret: + secretName: crossplane-provider-provider-ceph-server-cert +``` + +Finaly, you have to apply `ValidatingWebhookConfiguration`. + +```bash +PROV_CEPH_VER=v0.0.32 kubectl apply -f https://github.com/linode/provider-ceph/blob/release-${PROV_CEPH_VER}/package/webhookconfigurations/manifests.yaml +``` + ## Contact - Slack: Join our [#provider-ceph](https://crossplane.slack.com/archives/C05RKQRNDHA) slack channel. diff --git a/e2e/tests/stable/provider-ceph/00-enable-webhook.yaml b/e2e/tests/stable/provider-ceph/00-enable-webhook.yaml new file mode 100644 index 00000000..7644dabb --- /dev/null +++ b/e2e/tests/stable/provider-ceph/00-enable-webhook.yaml @@ -0,0 +1,4 @@ +apiVersion: kuttl.dev/v1beta1 +kind: TestStep +commands: + - command: kubectl apply -R -f ../../../../package/webhookconfigurations diff --git a/hack/deploy-provider.sh b/hack/deploy-provider.sh index bf6b3edf..3de72b45 100755 --- a/hack/deploy-provider.sh +++ b/hack/deploy-provider.sh @@ -5,12 +5,40 @@ : "${ARCH?= required}" : "${VERSION?= required}" +# Apply cert-manager Issuer and Certificate +kubectl apply -f - <