Ability to run webhook locally (#166)

Co-authored-by: Richard Kovacs <[email protected]>

.gitignore

@@ -1,7 +1,6 @@
 /bin
 /.cache
 /.work
-/.tmp-earthly*
 /_output
 cover.out
 /vendor

.husky/husky.mk

@@ -1,7 +1,11 @@
-
 HUSKY_VERSION ?= v0.2.16
 HUSKY ?= $(TOOLS_HOST_DIR)/husky-$(HUSKY_VERSION)
-husky: ## Download husky locally if necessary.
+
+husky.install: $(HUSKY)
+	@$(HUSKY) install
+
+## Download husky locally if necessary.
+$(HUSKY):
 	@$(INFO) installing husky $(HUSKY_VERSION)
 	@mkdir -p $(TOOLS_HOST_DIR)
 	@GOBIN=$(TOOLS_HOST_DIR) go install github.com/automation-co/husky@$(HUSKY_VERSION)

.mirrord/mirrord.mk

@@ -1,7 +1,14 @@
-MIRRORD_VERSION ?= 3.70.0
+MIRRORD_VERSION ?= 3.88.0
 MIRRORD := $(TOOLS_HOST_DIR)/mirrord-$(MIRRORD_VERSION)
 
-# mirrord download and install
+# Best for development - locally run provider-ceph controller.
+mirrord.cluster: dev-cluster crossplane-cluster load-package
+
+mirrord.run: $(MIRRORD)
+	@$(INFO) Starting mirrord on deployment
+	$(MIRRORD) exec -f .mirrord/mirrord.json make run
+
+# Download mirrord locally if necessary.
 $(MIRRORD):
 	@$(INFO) installing mirrord $(MIRRORD_VERSION)
 	@mkdir -p $(TOOLS_HOST_DIR) || $(FAIL)

Makefile

@@ -17,6 +17,15 @@ REPO ?= provider-ceph
 
 KUTTL_VERSION ?= 0.15.0
 
+CROSSPLANE_VERSION ?= 1.14.6
+
+# For local development, usually IP of docker0.
+WEBHOOK_HOST ?= 172.17.0.1
+# For local development, port of localtunnel.
+WEBHOOK_TUNNEL_PORT ?= 9999
+# For local development, subdomain of locatunnel.
+WEBHOOK_SUBDOMAIN ?= $(PROJECT_NAME)-$(shell git rev-parse --short HEAD)-$(shell date +%s)
+
 # ====================================================================================
 # Setup Output
 
@@ -113,9 +122,7 @@ build.init: $(UP)
 run: go.build
 	@$(INFO) Running Crossplane locally out-of-cluster . . .
 	@# To see other arguments that can be provided, run the command with --help instead
-	@# TODO: Webhooks are not enabled for local run.
-	@# A workaround for tls certs is required.
-	$(GO_OUT_DIR)/provider --zap-devel
+	$(GO_OUT_DIR)/provider --zap-devel --webhook-tls-cert-dir=$(PWD)/bin/certs --webhook-host=$(WEBHOOK_HOST)
 
 # Spin up a Kind cluster and localstack.
 # Create k8s service to allows pods to communicate with
@@ -132,7 +139,7 @@ crossplane-cluster: $(HELM3) cluster
 	@$(INFO) Installing Crossplane
 	@$(HELM3) repo add crossplane-stable https://charts.crossplane.io/stable
 	@$(HELM3) repo update
-	@$(HELM3) install crossplane --namespace crossplane-system --create-namespace crossplane-stable/crossplane
+	@$(HELM3) install crossplane --namespace crossplane-system --create-namespace --version $(CROSSPLANE_VERSION) crossplane-stable/crossplane
 	@$(OK) Installing Crossplane
 
 # Generate the provider-ceph package and webhookconfiguration manifest.
@@ -145,9 +152,9 @@ check-diff-pkg: generate-pkg
 
 # Kustomize the webhookconfiguration manifest that is created by 'generate' target.
 kustomize-webhook: $(KUSTOMIZE)
-	@cp $(XPKG_DIR)/webhookconfigurations/manifests.yaml $(VAL_WBHK_STAGE)
-	@$(KUSTOMIZE) build $(VAL_WBHK_STAGE) -o $(XPKG_DIR)/webhookconfigurations/manifests.yaml
-	@rm $(VAL_WBHK_STAGE)/manifests.yaml
+	@cp -f $(XPKG_DIR)/webhookconfigurations/manifests.yaml $(VAL_WBHK_STAGE)
+	@cp -f $(VAL_WBHK_STAGE)/service-patch-prod.yaml $(VAL_WBHK_STAGE)/service-patch.yaml
+	$(KUSTOMIZE) build $(VAL_WBHK_STAGE) -o $(XPKG_DIR)/webhookconfigurations/manifests.yaml
 
 # Build the controller image and the provider package.
 # Load the controller image to the Kind cluster and add the provider package
@@ -184,26 +191,56 @@ ceph-kuttl: $(KIND) $(KUTTL) $(HELM3) cluster-clean
 # containerised Crossplane componenets).
 # Install local provider-ceph CRDs.
 # Create ProviderConfig CR representing localstack.
-dev-cluster: $(KUBECTL) cluster
+dev-cluster: $(KUBECTL) generate-pkg generate-tests cluster
+	@rm -rf $(PWD)/bin/certs ; mkdir $(PWD)/bin/certs
+	@docker cp local-dev-control-plane:/etc/kubernetes/pki/ca.key $(PWD)/bin/certs/ca.key
+	@docker cp local-dev-control-plane:/etc/kubernetes/pki/ca.crt $(PWD)/bin/certs/ca.crt
+	@chmod 400 $(PWD)/bin/certs/ca.crt
+
+	@$(INFO) Generate TLS certificate for webhook.
+	@openssl genrsa \
+		-out $(PWD)/bin/certs/tls.key \
+		1024
+	@openssl req \
+		-new \
+		-key $(PWD)/bin/certs/tls.key \
+		-out $(PWD)/bin/certs/tls.csr \
+		-subj "/CN=linode.com"
+	@openssl x509 \
+		-req \
+		-days 365 \
+		-in $(PWD)/bin/certs/tls.csr \
+		-out $(PWD)/bin/certs/tls.crt \
+		-extfile <(printf "subjectAltName = IP:$(WEBHOOK_HOST)") \
+		-CAcreateserial \
+		-CA $(PWD)/bin/certs/ca.crt \
+		-CAkey $(PWD)/bin/certs/ca.key
+	@$(OK) Generate TLS certificate for webhook.
+
+	@$(INFO) Rendering webhook manifest.
+	@ex $(VAL_WBHK_STAGE)/service-patch-dev.tpl.yaml \
+		-c '%s/#WEBHOOK_HOST#/$(WEBHOOK_SUBDOMAIN).loca.lt/' \
+		-c 'sav! $(VAL_WBHK_STAGE)/service-patch.yaml' \
+		-c 'q' >/dev/null
+	$(KUSTOMIZE) build $(VAL_WBHK_STAGE) -o $(XPKG_DIR)/webhookconfigurations/manifests.yaml
+	@$(OK) Rendering webhook manifest.
+
 	@$(INFO) Installing CRDs, ProviderConfig and Localstack
-	@$(KUBECTL) apply -k https://github.com/crossplane/crossplane//cluster?ref=release-1.14
+	@$(KUBECTL) apply -k https://github.com/crossplane/crossplane/cluster?ref=v$(CROSSPLANE_VERSION)
 	@$(KUBECTL) apply -R -f package/crds
-	@# TODO: apply package/webhookconfigurations when webhooks can be enabled locally.
+	@$(KUBECTL) apply -R -f package/webhookconfigurations
 	@$(KUBECTL) apply -R -f e2e/localstack/localstack-deployment.yaml
 	@$(KUBECTL) apply -R -f e2e/localstack/localstack-provider-cfg-host.yaml
 	@$(OK) Installing CRDs and ProviderConfig
 
+	@$(INFO) Starting webhook tunnel.
+	@WEBHOOK_TUNNEL_PORT=$(WEBHOOK_TUNNEL_PORT) WEBHOOK_HOST=$(WEBHOOK_HOST) WEBHOOK_SUBDOMAIN=$(WEBHOOK_SUBDOMAIN) docker compose -f ./hack/localtunnel.yaml up -d
+	@$(OK) Starting webhook tunnel.
+
 # Best for development - locally run provider-ceph controller.
 # Removes need for Crossplane install via Helm.
 dev: dev-cluster run
 
-# Best for development - locally run provider-ceph controller.
-mirrord: dev-cluster crossplane-cluster load-package
-
-mirrord-run:
-	@$(INFO) Starting mirrord on deployment
-	$(MIRRORD) exec -f .mirrord/mirrord.json make run
-
 # Destroy Kind cluster and localstack.
 cluster-clean: $(KIND) $(KUBECTL)
 	@$(INFO) Deleting kind cluster

README.md

@@ -10,6 +10,7 @@ with the following features:
 - A controller that observes `Bucket` objects and reconciles these objects with the S3 backends.
 
 ## Developing
+
 Spin up the test environment, but with `provider-ceph` running locally in your terminal:
 
 ```
@@ -22,6 +23,15 @@ After you've made some changes, kill (Ctrl+C) the existing `provider-ceph` and r
 make run
 ```
 
+### Debugging
+Spin up the test environment, but with `provider-ceph` running locally in your terminal:
+
+```
+make mirrord.cluster mirrord.run
+```
+
+For debugging please install `mirrord` plugin in your IDE of choice.
+
 Refer to Crossplane's [CONTRIBUTING.md] file for more information on how the
 Crossplane community prefers to work. The [Provider Development][provider-dev]
 guide may also be of use.

cmd/provider/main.go

@@ -101,6 +101,7 @@ func main() {
 
 		assumeRoleArn = app.Flag("assume-role-arn", "Assume role ARN to be used for STS authentication").Default("").Envar("ASSUME_ROLE_ARN").String()
 
+		webhookHost       = app.Flag("webhook-host", "The host of the webhook server.").Default("0.0.0.0").Envar("WEBHOOK_HOST").String()
 		webhookTLSCertDir = app.Flag("webhook-tls-cert-dir", "The directory of TLS certificate that will be used by the webhook server. There should be tls.crt and tls.key files.").Default("/").Envar("WEBHOOK_TLS_CERT_DIR").String()
 		_                 = app.Flag("enable-validation-webhooks", "Enable support for Webhooks. [Deprecated, has no effect]").Default("false").Bool()
 	)
@@ -209,7 +210,6 @@ func main() {
 	kingpin.FatalIfError(err, "Cannot create HTTP client")
 
 	cacheHTTPClient.Timeout = *syncTimeout
-
 	mgr, err := ctrl.NewManager(cfg, ctrl.Options{
 		LeaderElection:             *leaderElection,
 		LeaderElectionID:           "crossplane-leader-election-provider-ceph-ibyaiby",
@@ -218,8 +218,11 @@ func main() {
 		RenewDeadline:              leaderRenew,
 		LeaseDuration:              &leaseDuration,
 		RetryPeriod:                &leaderRetryDuration,
-		WebhookServer:              webhook.NewServer(webhook.Options{CertDir: *webhookTLSCertDir}),
-		Scheme:                     providerSCheme,
+		WebhookServer: webhook.NewServer(webhook.Options{
+			Host:    *webhookHost,
+			CertDir: *webhookTLSCertDir,
+		}),
+		Scheme: providerSCheme,
 		Cache: kcache.Options{
 			HTTPClient: cacheHTTPClient,
 			SyncPeriod: syncInterval,

hack/localtunnel.yaml

@@ -0,0 +1,18 @@
+version: '3'
+
+services:
+  # Socat proxies localtunnel requests to webhook service
+  provider-ceph-socat:
+    image: nixery.dev/socat
+    command: socat TCP-LISTEN:${WEBHOOK_TUNNEL_PORT},reuseaddr,fork OPENSSL:${WEBHOOK_HOST}:9443,verify=0
+    network_mode: "host"
+    container_name: provider-ceph-socat
+    restart: unless-stopped
+
+  # Localtunnel creates an external domain with valid certificate
+  provider-ceph-localtunnel:
+    image: nixery.dev/nodepackages.localtunnel
+    command: lt --subdomain ${WEBHOOK_SUBDOMAIN} --port ${WEBHOOK_TUNNEL_PORT}
+    network_mode: "host"
+    container_name: provider-ceph-localtunnel
+    restart: unless-stopped

package/webhookconfigurations/manifests.yaml

@@ -7,9 +7,10 @@ webhooks:
   - v1
   clientConfig:
     service:
-      name: webhook-service
-      namespace: system
+      name: provider-ceph
+      namespace: crossplane-system
       path: /validate-provider-ceph-ceph-crossplane-io-v1alpha1-bucket
+      port: 9443
   failurePolicy: Fail
   name: bucket-validation.providerceph.crossplane.io
   objectSelector:

staging/validatingwebhookconfiguration/.gitignore

@@ -0,0 +1 @@
+manifests.yaml

staging/validatingwebhookconfiguration/kustomization.yaml

@@ -2,6 +2,9 @@ resources:
 - manifests.yaml
 
 patches:
-- path: patch.yaml
+- path: object-selector-patch.yaml
+  target:
+    kind: ValidatingWebhookConfiguration
+- path: service-patch.yaml
   target:
     kind: ValidatingWebhookConfiguration

staging/validatingwebhookconfiguration/patch.yaml → staging/validatingwebhookconfiguration/object-selector-patch.yaml

@@ -6,4 +6,4 @@ webhooks:
 - name: bucket-validation.providerceph.crossplane.io
   objectSelector:
     matchLabels:
-      provider-ceph.crossplane.io/validation-required: "true" 
+      provider-ceph.crossplane.io/validation-required: "true"

staging/validatingwebhookconfiguration/service-patch-dev.tpl.yaml

@@ -0,0 +1,5 @@
+- op: remove
+  path: /webhooks/0/clientConfig/service
+- op: add
+  path: /webhooks/0/clientConfig/url
+  value: https://#WEBHOOK_HOST#/validate-provider-ceph-ceph-crossplane-io-v1alpha1-bucket

staging/validatingwebhookconfiguration/service-patch-prod.yaml

@@ -0,0 +1,11 @@
+apiVersion: admissionregistration.k8s.io/v1
+kind: ValidatingWebhookConfiguration
+metadata:
+  name: validating-webhook-configuration
+webhooks:
+- name: bucket-validation.providerceph.crossplane.io
+  clientConfig:
+    service:
+      name: provider-ceph
+      namespace: crossplane-system
+      port: 9443

staging/validatingwebhookconfiguration/service-patch.yaml

@@ -0,0 +1,11 @@
+apiVersion: admissionregistration.k8s.io/v1
+kind: ValidatingWebhookConfiguration
+metadata:
+  name: validating-webhook-configuration
+webhooks:
+- name: bucket-validation.providerceph.crossplane.io
+  clientConfig:
+    service:
+      name: provider-ceph
+      namespace: crossplane-system
+      port: 9443