ability to create operate ingress with sticky sessions and scale impo…

…rters

.gitignore

@@ -44,5 +44,6 @@ google/multi-region/active-active/generated
 **/tasklist-service-webapp.yaml
 **/tasklist-tasklist-ingress.yaml
 **/tasklist-camunda-ingress.yaml
+**/operate-ingress.yaml
 
 

high-available-webapps/Makefile

@@ -25,57 +25,79 @@ all: camunda await-zeebe scale-operate scale-tasklist scale-optimize await-ident
 .PHONY: clean
 clean: clean-operate clean-tasklist clean-optimize clean-camunda
 
-.PHONY: scale-operate-webapp
-scale-operate-webapp:
-	kubectl get deployment $(release)-operate -n $(namespace) -o yaml > operate_template.yaml
-	yq eval 'del(.status, .metadata, .spec.template.metadata.annotations, .spec.template.metadata.creationTimestamp,  .spec.selector)' -i operate_template.yaml
-	yq eval '.metadata.name = "operate-webapp" | .spec.selector.matchLabels.webapp = "operate-webapp" | .spec.template.metadata.labels.webapp = "operate-webapp" | .spec.replicas = $(replicas)' -i operate_template.yaml
-	yq eval '.spec.template.spec.containers[0].env += [{"name": "CAMUNDA_OPERATE_IMPORTERENABLED", "value": "FALSE"}, {"name": "CAMUNDA_OPERATE_ARCHIVERENABLED", "value": "FALSE"}]' operate_template.yaml -i
-	kubectl apply -f operate_template.yaml
-	rm operate_template.yaml
+.PHONY: remove-operate-from-ingress
+remove-operate-from-ingress:
+	kubectl get ingress $(release)-camunda-platform -n $(namespace) -o yaml > ingress-camunda-platform.yaml
+	yq eval 'del(.spec.rules[0].http.paths[] | select(.backend.service.name == "camunda-operate") )' -i ingress-camunda-platform.yaml
+	kubectl apply -f ingress-camunda-platform.yaml -n $(namespace)
+	rm ingress-camunda-platform.yaml
+
+operate-ingress.yaml: fqdn
+	sed "s/YOUR_HOSTNAME/$(fqdn)/g;" $(root)/high-available-webapps/operate-ingress.tpl.yaml > ./operate-ingress.yaml;
+
+.PHONY: create-operate-ingress
+create-operate-ingress: remove-operate-from-ingress operate-ingress.yaml
+	kubectl apply -f ./operate-ingress.yaml -n $(namespace)
+	rm ./operate-ingress.yaml
 
 .PHONY: scale-operate-webapp-yaml
 scale-operate-webapp-yaml:
 	kubectl get deployment $(release)-operate -n $(namespace) -o yaml > operate_template.yaml
 	yq eval 'del(.status, .metadata, .spec.template.metadata.annotations, .spec.template.metadata.creationTimestamp,  .spec.selector)' -i operate_template.yaml
 	yq eval '.metadata.name = "operate-webapp" | .spec.selector.matchLabels.webapp = "operate-webapp" | .spec.template.metadata.labels.webapp = "operate-webapp" | .spec.replicas = $(replicas)' -i operate_template.yaml
-	yq eval '.spec.template.spec.containers[0].env += [{"name": "CAMUNDA_OPERATE_IMPORTERENABLED", "value": "FALSE"}, {"name": "CAMUNDA_OPERATE_ARCHIVERENABLED", "value": "FALSE"}]' operate_template.yaml -i
-#	kubectl apply -f operate_template.yaml
-#	rm operate_template.yaml
+	yq eval '.spec.template.spec.containers[0].env += [{"name": "CAMUNDA_OPERATE_WEBAPPENABLED", "value": "FALSE"}, {"name": "CAMUNDA_OPERATE_IMPORTERENABLED", "value": "FALSE"}, {"name": "CAMUNDA_OPERATE_ARCHIVERENABLED", "value": "FALSE"}]' operate_template.yaml -i
 
-.PHONY: scale-operate-importer0
-scale-operate-importer0:
-#	kubectl delete deployment $(release)-operate -n $(namespace)
-	kubectl get deployment $(release)-operate -n $(namespace) -o yaml > operate_template-import0.yaml
+.PHONY: scale-operate-webapp
+scale-operate-webapp: scale-operate-webapp-yaml
+	kubectl apply -f operate_template.yaml
+	rm operate_template.yaml
+
+.PHONY: scale-operate-importer0-yaml
+scale-operate-importer0-yaml:
+	kubectl get deployment operate-webapp -n $(namespace) -o yaml > operate_template-import0.yaml
 	yq eval 'del(.status, .metadata, .spec.template.metadata.annotations, .spec.template.metadata.creationTimestamp,  .spec.selector)' -i operate_template-import0.yaml
 	yq eval '.metadata.name = "operate-import0" | .spec.selector.matchLabels.webapp = "operate-import0" | .spec.template.metadata.labels.webapp = "operate-import0" | .spec.replicas = 1' -i operate_template-import0.yaml
+	yq eval 'del (.spec.template.spec.containers[0].env.[] | select(.name == "CAMUNDA_OPERATE_WEBAPPENABLED") )' operate_template-import0.yaml -i
+	yq eval 'del (.spec.template.spec.containers[0].env.[] | select(.name == "CAMUNDA_OPERATE_IMPORTERENABLED") )' operate_template-import0.yaml -i
+	yq eval 'del (.spec.template.spec.containers[0].env.[] | select(.name == "CAMUNDA_OPERATE_ARCHIVERENABLED") )' operate_template-import0.yaml -i
 	yq eval '.spec.template.spec.containers[0].env += [{"name": "CAMUNDA_OPERATE_WEBAPPENABLED", "value": "FALSE"}, {"name": "CAMUNDA_OPERATE_IMPORTERENABLED", "value": "TRUE"}, {"name": "CAMUNDA_OPERATE_ARCHIVERENABLED", "value": "TRUE"}, {"name": "CAMUNDA_OPERATE_CLUSERNODE_NODECOUNT", "value": "3"}, {"name": "CAMUNDA_OPERATE_CLUSERNODE_CURRENTNODEID", "value": "0"} ]' operate_template-import0.yaml -i
-#	kubectl apply -f operate_template.yaml
-#	rm operate_template.yaml
 
-.PHONY: scale-operate-importer1
-scale-operate-importer1:
-#	kubectl delete deployment $(release)-operate -n $(namespace)
-	kubectl get deployment $(release)-operate -n $(namespace) -o yaml > operate_template-import1.yaml
+.PHONY: scale-operate-importer0
+scale-operate-importer0: scale-operate-importer0-yaml
+	-kubectl delete deployment $(release)-operate -n $(namespace)
+	kubectl apply -f operate_template-import0.yaml
+	rm operate_template-import0.yaml
+
+.PHONY: scale-operate-importer1-yaml
+scale-operate-importer1-yaml:
+	kubectl get deployment operate-webapp -n $(namespace) -o yaml > operate_template-import1.yaml
 	yq eval 'del(.status, .metadata, .spec.template.metadata.annotations, .spec.template.metadata.creationTimestamp,  .spec.selector)' -i operate_template-import1.yaml
 	yq eval '.metadata.name = "operate-import1" | .spec.selector.matchLabels.webapp = "operate-import1" | .spec.template.metadata.labels.webapp = "operate-import1" | .spec.replicas = 1' -i operate_template-import1.yaml
+	yq eval 'del (.spec.template.spec.containers[0].env.[] | select(.name == "CAMUNDA_OPERATE_WEBAPPENABLED") )' operate_template-import1.yaml -i
+	yq eval 'del (.spec.template.spec.containers[0].env.[] | select(.name == "CAMUNDA_OPERATE_IMPORTERENABLED") )' operate_template-import1.yaml -i
+	yq eval 'del (.spec.template.spec.containers[0].env.[] | select(.name == "CAMUNDA_OPERATE_ARCHIVERENABLED") )' operate_template-import1.yaml -i
 	yq eval '.spec.template.spec.containers[0].env += [{"name": "CAMUNDA_OPERATE_WEBAPPENABLED", "value": "FALSE"}, {"name": "CAMUNDA_OPERATE_IMPORTERENABLED", "value": "TRUE"}, {"name": "CAMUNDA_OPERATE_ARCHIVERENABLED", "value": "TRUE"}, {"name": "CAMUNDA_OPERATE_CLUSERNODE_NODECOUNT", "value": "3"}, {"name": "CAMUNDA_OPERATE_CLUSERNODE_CURRENTNODEID", "value": "1"} ]' operate_template-import1.yaml -i
-#	kubectl apply -f operate_template.yaml
-#	rm operate_template.yaml
 
-.PHONY: scale-operate-importer2
-scale-operate-importer2:
-#	kubectl delete deployment $(release)-operate -n $(namespace)
-	kubectl get deployment $(release)-operate -n $(namespace) -o yaml > operate_template-import2.yaml
+.PHONY: scale-operate-importer1
+scale-operate-importer1: scale-operate-importer1-yaml
+	-kubectl delete deployment $(release)-operate -n $(namespace)
+	kubectl apply -f operate_template-import1.yaml
+	rm operate_template-import1.yaml
+
+.PHONY: scale-operate-importer2-yaml
+scale-operate-importer2-yaml:
+	kubectl get deployment operate-webapp -n $(namespace) -o yaml > operate_template-import2.yaml
 	yq eval 'del(.status, .metadata, .spec.template.metadata.annotations, .spec.template.metadata.creationTimestamp,  .spec.selector)' -i operate_template-import2.yaml
 	yq eval '.metadata.name = "operate-import2" | .spec.selector.matchLabels.webapp = "operate-import2" | .spec.template.metadata.labels.webapp = "operate-import2" | .spec.replicas = 1' -i operate_template-import2.yaml
+	yq eval 'del (.spec.template.spec.containers[0].env.[] | select(.name == "CAMUNDA_OPERATE_WEBAPPENABLED") )' operate_template-import2.yaml -i
+	yq eval 'del (.spec.template.spec.containers[0].env.[] | select(.name == "CAMUNDA_OPERATE_IMPORTERENABLED") )' operate_template-import2.yaml -i
+	yq eval 'del (.spec.template.spec.containers[0].env.[] | select(.name == "CAMUNDA_OPERATE_ARCHIVERENABLED") )' operate_template-import2.yaml -i
 	yq eval '.spec.template.spec.containers[0].env += [{"name": "CAMUNDA_OPERATE_WEBAPPENABLED", "value": "FALSE"}, {"name": "CAMUNDA_OPERATE_IMPORTERENABLED", "value": "TRUE"}, {"name": "CAMUNDA_OPERATE_ARCHIVERENABLED", "value": "TRUE"}, {"name": "CAMUNDA_OPERATE_CLUSERNODE_NODECOUNT", "value": "3"}, {"name": "CAMUNDA_OPERATE_CLUSERNODE_CURRENTNODEID", "value": "2"} ]' operate_template-import2.yaml -i
-#	kubectl apply -f operate_template.yaml
-#	rm operate_template.yaml
 
-# CAMUNDA_OPERATE_CLUSTERNODE_PARTITIONIDS
-# CAMUNDA_OPERATE_CLUSERNODE_NODECOUNT
-# CAMUNDA_OPERATE_CLUSERNODE_CURRENTNODEID
+.PHONY: scale-operate-importer2
+scale-operate-importer2: scale-operate-importer2-yaml
+	kubectl apply -f operate_template-import2.yaml
+	rm operate_template-import2.yaml
 
 .PHONY: scale-tasklist
 scale-tasklist:
@@ -117,4 +139,5 @@ clean-tasklist:
 clean-optimize:
 	kubectl delete deployment optimize-webapp -n $(namespace)
 
-include $(root)/include/camunda.mk
+include $(root)/include/camunda.mk
+include $(root)/include/ingress-nginx.mk

high-available-webapps/README.md

@@ -42,9 +42,21 @@ Compatibility: This Makefile was tested against version 8.3.
 
 2. **Execute the Commands**:
    - Run `make all` to setup camunda and scale Operate,Tasklist, Optimize and Identity afterwards
-   - Run `make scale-operate` to scale the Operate component.
+   - Run `make scale-operate-webapp` to scale the Operate component.
    - Run `make scale-tasklist` to scale the Tasklist component.
    - Run `make scale-optimize` to scale the Optimize component.
    - Run `make scale-identity` to scale the Identity component.
+
+3. **Optional** The following can be used in 3 partition environment to create a separate Operate Importer/Archiver for each partition: 
+   - Run `make scale-operate-importer0 scale-operate-importer1 scale-operate-importer2` 
 
 **Note**: Before executing these commands, ensure you have proper access to the Kubernetes cluster and the necessary permissions for scaling deployments.
+
+## Sticky Sessions
+
+When a webapp deployment (operate, optimize, or operate) has multiple replicas, each authenticated session must be
+"pinned" to the same replica. Otherwise, each request will be routed to a different replica and an infinite login redirect may occur. 
+
+See the [Makefile](./Makefile) and take a look at the target named `create-operate-ingress` to understand how to remove
+an ingress rule for operate and create a separate ingress that has sticky sessions configured using cookie annotations.  
+

high-available-webapps/operate-ingress.tpl.yaml

@@ -0,0 +1,31 @@
+---
+# Source: camunda-platform/templates/ingress.yaml
+apiVersion: networking.k8s.io/v1
+kind: Ingress
+metadata:
+  name: operate-ingress
+  labels:
+    app.kubernetes.io/app: operate-ingress
+  annotations:
+    ingress.kubernetes.io/rewrite-target: /
+    nginx.ingress.kubernetes.io/ssl-redirect: "false"
+    nginx.ingress.kubernetes.io/affinity: "cookie"
+    nginx.ingress.kubernetes.io/session-cookie-name: "operate-route"
+    nginx.ingress.kubernetes.io/session-cookie-max-age: "172800"
+spec:
+  ingressClassName: nginx
+  rules:
+    - host: YOUR_HOSTNAME
+      http:
+        paths:
+          - backend:
+              service:
+                name: camunda-operate
+                port:
+                  number: 80
+            path: /operate
+            pathType: Prefix
+  tls:
+    - hosts:
+        - YOUR_HOSTNAME
+      secretName: tls-secret