From 0877f7dd17f1e2cf852d391995736af5ebdd7f22 Mon Sep 17 00:00:00 2001
From: Martin Gruner <mg@zammad.com>
Date: Fri, 26 Apr 2024 07:06:10 +0200
Subject: [PATCH] Fixes: #236 - Running Zammad with replicas > 1 (#243)

- Splits up the chart from one StatefulSet into 4 Deployments and one Job.
  - The nginx and railsserver Deployments are freely scalable, the scheduler and websocket must remain at replicas: 1
  - The Job will be re-created on any chart update (via uuid in the name) and run the migrations. Deployments will fail until migrations are executed. This greatly reduces start-up and update downtime.
- Refactor some code into helper templates to reduce redundancy / improve maintainability.
- Storage requirements changed. Please read the updating instructions carefully before updating.

Special thanks to @klml and @monotek for guidance and feedback.
---
 .github/workflows/ci.yaml                     |   4 +-
 zammad/Chart.yaml                             |   2 +-
 zammad/README.md                              |  54 +-
 .../{full-secrets.yaml => full-objects.yaml}  |  13 +-
 zammad/ci/full-values.yaml                    |   3 +
 zammad/templates/_helpers.tpl                 | 166 ++++++
 zammad/templates/configmap-nginx.yaml         |   8 +-
 zammad/templates/deployment-nginx.yaml        |  60 +++
 zammad/templates/deployment-railsserver.yaml  |  49 ++
 zammad/templates/deployment-scheduler.yaml    |  41 ++
 zammad/templates/deployment-websocket.yaml    |  48 ++
 zammad/templates/ingress.yaml                 |   4 +-
 zammad/templates/job-init.yaml                |  93 ++++
 .../{service.yaml => service-nginx.yaml}      |   3 +-
 zammad/templates/service-railsserver.yaml     |  15 +
 zammad/templates/service-websocket.yaml       |  15 +
 zammad/templates/statefulset.yaml             | 485 ------------------
 zammad/templates/tests/test-connection.yaml   |   2 +-
 zammad/values.yaml                            |  66 +--
 19 files changed, 574 insertions(+), 557 deletions(-)
 rename zammad/ci/{full-secrets.yaml => full-objects.yaml} (90%)
 create mode 100644 zammad/templates/deployment-nginx.yaml
 create mode 100644 zammad/templates/deployment-railsserver.yaml
 create mode 100644 zammad/templates/deployment-scheduler.yaml
 create mode 100644 zammad/templates/deployment-websocket.yaml
 create mode 100644 zammad/templates/job-init.yaml
 rename zammad/templates/{service.yaml => service-nginx.yaml} (77%)
 create mode 100644 zammad/templates/service-railsserver.yaml
 create mode 100644 zammad/templates/service-websocket.yaml
 delete mode 100644 zammad/templates/statefulset.yaml

diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml
index ce4d28a3..9c03eeb4 100644
--- a/.github/workflows/ci.yaml
+++ b/.github/workflows/ci.yaml
@@ -124,8 +124,8 @@ jobs:
       - name: Create Namespace 'zammad'
         run: kubectl create namespace zammad
 
-      - name: Install secrets
-        run: kubectl create --namespace zammad --filename zammad/ci/full-secrets.yaml
+      - name: Install additional objects for 'full' test scenario
+        run: kubectl create --namespace zammad --filename zammad/ci/full-objects.yaml
 
       - name: Run chart-testing (install)
         run: ct install --config .github/ct.yaml --helm-extra-args '--timeout 900s'
\ No newline at end of file
diff --git a/zammad/Chart.yaml b/zammad/Chart.yaml
index 0e880684..1ffbe7b2 100644
--- a/zammad/Chart.yaml
+++ b/zammad/Chart.yaml
@@ -1,6 +1,6 @@
 apiVersion: v2
 name: zammad
-version: 11.0.0
+version: 12.0.0
 appVersion: 6.3.0
 description: Zammad is a web based open source helpdesk/customer support system with many features to manage customer communication via several channels like telephone, facebook, twitter, chat and e-mails.
 home: https://zammad.org
diff --git a/zammad/README.md b/zammad/README.md
index a2370a79..e2e54e7a 100644
--- a/zammad/README.md
+++ b/zammad/README.md
@@ -31,6 +31,15 @@ helm repo add zammad https://zammad.github.io/zammad-helm
 helm upgrade --install zammad zammad/zammad
 ```
 
+Once the Zammad pod is ready, it can be accessed using the ingress or port forwarding.
+To use port forwarding:
+
+```console
+kubectl port-forward service/zammad-nginx 8080
+```
+
+Now you can open <http://localhost:8080> in your browser.
+
 ## Uninstalling the Chart
 
 To remove the chart again use the following:
@@ -58,15 +67,8 @@ Only if you have a large volume of tickets and attachments, you may need to stor
 
 We recommend the `S3` storage provider using the optional `minio` subchart in this case.
 
-You can also use `File` storage. In this case, you should check:
-
-- If you already use an `externalVolumeClaim` with `ReadWriteMany` access, you can keep using that.
-- If you already use an `externalVolumeClaim` with another access mode, we recommend migrating to S3 storage (see below).
-- If you used the default `PVC` of the Zammad `StatefulSet`, we also recommend migrating to S3 storage (see below).
-
-Background information: a future version of Zammad will increase the scalability by splitting up the current `Statefulset`
-into several `Deployment`s which can be scaled. This means the `PVC` of the current `StatefulSet` will then not be usable any more,
-and any volumes will have to support `ReadWriteMany` access.
+You can also use `File` storage. In this case, you need to provide an existing `PVC` via `zammadConfig.storageVolume`.
+Note that this `PVC` must provide `ReadWriteMany` access to work properly for the different Deployments which may be on different nodes.
 
 #### How to migrate from `File` to `S3` storage
 
@@ -102,11 +104,11 @@ zammadConfig:
     zammad:
       securityContext:
         runAsUser: null
-      customInit: |
-        # use an openshift uid owned /tmp for attachments upload
-        mkdir -pv /opt/zammad/var/tmp && chmod -v +t /opt/zammad/var/tmp
-  railsserver:
-    tmpdir: "/opt/zammad/var/tmp"
+  volumePermissions:
+    enabled: false
+  tmpDirVolume:
+    emptyDir:
+      medium: memory
 
 elasticsearch:
   sysctlImage:
@@ -142,18 +144,26 @@ redis:
       enabled: false
 ```
 
-## Using Zammad
+## Upgrading
 
-Once the Zammad pod is ready, it can be accessed using the ingress or port forwarding.
-To use port forwarding:
+### From Chart Version 11.x to 12.0.0
 
-```console
-kubectl port-forward service/zammad 8080
-```
+#### The Previous `StatefulSet` Was Split up into `Deployments`
 
-Now you can open <http://localhost:8080> in your browser.
+- `replicas` can be set independently now for `zammad-nginx` and `zammad-railsserver`, allowing free scaling and HA setup for these.
+  - For `zammad-scheduler` and `zammad-websocket`, `replicas` is fixed to `1` as they may only run once in the cluster.
+- The `initContainers` moved to a new `zammad-init` `Job` now which will be run on every `helm upgrade`. This reduces startup time greatly.
+- The nginx `Service` was renamed from `zammad` to `zammad-nginx`.
+- The previous `Values.sidecars` setting does not exist any more. Instead, you need to specify sidecars now on a per deployment basis, e.g. `Values.zammadConfig.scheduler.sidecars`.
 
-## Upgrading
+#### Storage Requirements Changed
+
+- If you use the default `DB` or the new `S3` storage backend for file storage, you don't need to do anything.
+- If you use the `File` storage backend instead, Zammad now requires a `ReadWriteMany` volume for `storage/` that is shared in the cluster.
+  - If you already had one via `persistence.existingClaim` before, you need to ensure it has `ReadWriteMany` access to be mountable across nodes and provide it via `zammadConfig.storageVolume.existingClaim`.
+  - If you used the default `PersistentVolumeClaim` of the `StatefulSet`, you need to take manual action:
+    - You can either migrate to `S3` storage **before upgrading** to the new major version as described above in [Configuration](#how-to-migrate-from-file-to-s3-storage).
+    - Or you can provide a `zammadConfig.storageVolume.existingClaim` with `ReadWriteMany` permission and migrate your existing data to it from the old `StatefulSet`.
 
 ### From Chart Version 10.x to 11.0.0
 
diff --git a/zammad/ci/full-secrets.yaml b/zammad/ci/full-objects.yaml
similarity index 90%
rename from zammad/ci/full-secrets.yaml
rename to zammad/ci/full-objects.yaml
index c69e1ca4..f5fa959f 100644
--- a/zammad/ci/full-secrets.yaml
+++ b/zammad/ci/full-objects.yaml
@@ -37,4 +37,15 @@ data:
 kind: Secret
 metadata:
   name: autowizard
-type: Opaque
\ No newline at end of file
+type: Opaque
+---
+apiVersion: v1
+kind: PersistentVolumeClaim
+metadata:
+  name: storage-volume-claim
+spec:
+  accessModes:
+    - ReadWriteOnce   # Testing env does not provide ReadWrite Many, but for CI this is enough.
+  resources:
+    requests:
+      storage: 32Mi
\ No newline at end of file
diff --git a/zammad/ci/full-values.yaml b/zammad/ci/full-values.yaml
index 254cf8b8..720da7e7 100644
--- a/zammad/ci/full-values.yaml
+++ b/zammad/ci/full-values.yaml
@@ -33,5 +33,8 @@ redis:
     existingSecretPasswordKey: redis-password
 
 zammadConfig:
+  storageVolume:
+    enabled: true
+    existingClaim: 'storage-volume-claim'
   minio:
     enabled: true
diff --git a/zammad/templates/_helpers.tpl b/zammad/templates/_helpers.tpl
index d41a9eef..e5101494 100644
--- a/zammad/templates/_helpers.tpl
+++ b/zammad/templates/_helpers.tpl
@@ -135,3 +135,169 @@ S3 access URL
 {{- end -}}
 {{- end -}}
 {{- end -}}
+
+{{/*
+environment variables for the Zammad Rails stack
+*/}}
+{{- define "zammad.env" -}}
+{{- if or .Values.zammadConfig.redis.pass .Values.secrets.redis.useExisting -}}
+- name: REDIS_PASSWORD
+  valueFrom:
+    secretKeyRef:
+      name: {{ template "zammad.redisSecretName" . }}
+      key: {{ .Values.secrets.redis.secretKey }}
+{{- end }}
+- name: MEMCACHE_SERVERS
+  value: "{{ if .Values.zammadConfig.memcached.enabled }}{{ .Release.Name }}-memcached{{ else }}{{ .Values.zammadConfig.memcached.host }}{{ end }}:{{ .Values.zammadConfig.memcached.port }}"
+- name: RAILS_TRUSTED_PROXIES
+  value: "{{ .Values.zammadConfig.railsserver.trustedProxies }}"
+- name: REDIS_URL
+  value: "redis://:$(REDIS_PASSWORD)@{{ if .Values.zammadConfig.redis.enabled }}{{ .Release.Name }}-redis-master{{ else }}{{ .Values.zammadConfig.redis.host }}{{ end }}:{{ .Values.zammadConfig.redis.port }}"
+- name: POSTGRESQL_PASS
+  valueFrom:
+    secretKeyRef:
+      name: {{ template "zammad.postgresqlSecretName" . }}
+      key: {{ .Values.secrets.postgresql.secretKey }}
+- name: DATABASE_URL
+  value: "postgres://{{ .Values.zammadConfig.postgresql.user }}:$(POSTGRESQL_PASS)@{{ if .Values.zammadConfig.postgresql.enabled }}{{ .Release.Name }}-postgresql{{ else }}{{ .Values.zammadConfig.postgresql.host }}{{ end }}:{{ .Values.zammadConfig.postgresql.port }}/{{ .Values.zammadConfig.postgresql.db }}?{{ .Values.zammadConfig.postgresql.options }}"
+{{ include "zammad.env.S3_URL" . }}
+- name: TMP # All zammad containers need the possibility to create temporary files, e.g. for file uploads or image resizing.
+  value: {{ .Values.zammadConfig.railsserver.tmpdir }}
+{{- with .Values.extraEnv }}
+{{ toYaml . }}
+{{- end }}
+{{- if .Values.autoWizard.enabled }}
+- name: AUTOWIZARD_RELATIVE_PATH
+  value: tmp/auto_wizard/auto_wizard.json
+{{- end }}
+{{- end -}}
+
+{{/*
+environment variable to let Rails fail during startup if migrations are pending
+*/}}
+{{- define "zammad.env.failOnPendingMigrations" -}}
+# Let containers fail if migrations are pending.
+- name: RAILS_CHECK_PENDING_MIGRATIONS
+  value: 'true'
+{{- end -}}
+
+{{/*
+volume mounts for the Zammad Rails stack
+*/}}
+{{- define "zammad.volumeMounts" -}}
+- name: {{ template "zammad.fullname" . }}-tmp
+  mountPath: /tmp
+- name: {{ template "zammad.fullname" . }}-tmp
+  mountPath: /opt/zammad/tmp
+{{- if .Values.zammadConfig.storageVolume.enabled }}
+- name: {{ template "zammad.fullname" . }}-storage
+  mountPath: /opt/zammad/storage
+{{- end -}}
+{{- if .Values.autoWizard.enabled }}
+- name: autowizard
+  mountPath: "/opt/zammad/tmp/auto_wizard"
+{{- end }}
+{{- end -}}
+
+{{/*
+volumes for the Zammad Rails stack
+*/}}
+{{- define "zammad.volumes" -}}
+- name: {{ include "zammad.fullname" . }}-tmp
+  {{- toYaml .Values.zammadConfig.tmpDirVolume | nindent 2 }}
+{{- if .Values.zammadConfig.storageVolume.enabled }}
+{{- if .Values.zammadConfig.storageVolume.existingClaim }}
+- name: {{ template "zammad.fullname" . }}-storage
+  persistentVolumeClaim:
+    claimName: {{ .Values.zammadConfig.storageVolume.existingClaim | default (include "zammad.fullname" .) }}
+{{- else }}
+  {{ fail "Please provide an existing PersistentVolumeClaim with ReadWriteMany access if you enable .Values.zammadConfig.storageVolume." }}
+{{- end -}}
+{{- end -}}
+{{- if .Values.autoWizard.enabled }}
+- name: autowizard
+  secret:
+    secretName: {{ template "zammad.autowizardSecretName" . }}
+    items:
+    - key: {{ .Values.secrets.autowizard.secretKey }}
+      path: auto_wizard.json
+{{- end }}
+{{- end -}}
+
+{{/*
+shared configuration for Zammad Pods
+*/}}
+{{- define "zammad.podSpec" -}}
+{{- with .Values.image.imagePullSecrets }}
+imagePullSecrets:
+  {{- toYaml . | nindent 2 }}
+{{- end }}
+{{- if .Values.serviceAccount.create }}
+serviceAccountName: {{ include "zammad.serviceAccountName" . }}
+{{- end }}
+{{- with .Values.nodeSelector }}
+nodeSelector:
+  {{- toYaml . | nindent 2 }}
+{{- end }}
+{{- with .Values.affinity }}
+affinity:
+  {{- toYaml . | nindent 2 }}
+{{- end }}
+{{- with .Values.tolerations }}
+tolerations:
+  {{- toYaml . | nindent 2 }}
+{{- end }}
+{{- with .Values.securityContext }}
+securityContext:
+  {{- toYaml . | nindent 2 }}
+{{- end }}
+{{- end -}}
+
+{{/*
+shared configuration for Zammad Deployment Pods
+*/}}
+{{- define "zammad.podSpec.deployment" -}}
+{{ include "zammad.podSpec" . }}
+{{- if .Values.zammadConfig.initContainers.volumePermissions.enabled }}
+initContainers:
+  - name: zammad-volume-permissions
+    image: "{{ .Values.zammadConfig.initContainers.volumePermissions.image.repository }}:{{ .Values.zammadConfig.initContainers.volumePermissions.image.tag }}"
+    imagePullPolicy: {{ .Values.zammadConfig.initContainers.volumePermissions.image.pullPolicy }}
+    command:
+      {{- .Values.zammadConfig.initContainers.volumePermissions.command | toYaml | nindent 6 }}
+    {{- with .Values.zammadConfig.initContainers.volumePermissions.resources }}
+    resources:
+      {{- toYaml . | nindent 6 }}
+    {{- end }}
+    {{- with .Values.zammadConfig.initContainers.volumePermissions.securityContext }}
+    securityContext:
+      {{- toYaml . | nindent 6 }}
+    {{- end }}
+    volumeMounts:
+      {{- include "zammad.volumeMounts" . | nindent 6 }}
+{{- end }}
+{{- end -}}
+
+{{/*
+shared configuration for Zammad containers
+*/}}
+{{- define "zammad.containerSpec" -}}
+image: "{{ .Values.image.repository }}:{{ .Values.image.tag | default .Chart.AppVersion }}"
+imagePullPolicy: {{ .Values.image.pullPolicy }}
+{{- with .containerConfig.livenessProbe }}
+livenessProbe:
+  {{- toYaml . | nindent 2 }}
+{{- end }}
+{{- with .containerConfig.readinessProbe }}
+readinessProbe:
+  {{- toYaml . | nindent 2 }}
+{{- end }}
+{{- with .containerConfig.resources }}
+resources:
+  {{- toYaml . | nindent 2 }}
+{{- end }}
+{{- with .containerConfig.securityContext }}
+securityContext:
+  {{- toYaml . | nindent 2 }}
+{{- end }}
+{{- end -}}
\ No newline at end of file
diff --git a/zammad/templates/configmap-nginx.yaml b/zammad/templates/configmap-nginx.yaml
index 913e24c5..29e3edf2 100644
--- a/zammad/templates/configmap-nginx.yaml
+++ b/zammad/templates/configmap-nginx.yaml
@@ -13,11 +13,11 @@ data:
     server_tokens off;
 
     upstream zammad-railsserver {
-        server localhost:3000;
+        server {{ template "zammad.fullname" . }}-railsserver:3000;
     }
 
     upstream zammad-websocket {
-        server localhost:6042;
+        server {{ template "zammad.fullname" . }}-websocket:6042;
     }
 
     server {
@@ -106,9 +106,9 @@ data:
     }
   nginx.conf: |-
     worker_processes auto;
-    
+
     pid /tmp/nginx.pid;
-    
+
     include /etc/nginx/modules-enabled/*.conf;
 
     events {
diff --git a/zammad/templates/deployment-nginx.yaml b/zammad/templates/deployment-nginx.yaml
new file mode 100644
index 00000000..7cba7f5a
--- /dev/null
+++ b/zammad/templates/deployment-nginx.yaml
@@ -0,0 +1,60 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: {{ template "zammad.fullname" . }}-nginx
+  labels:
+    {{- include "zammad.labels" . | nindent 4 }}
+    app.kubernetes.io/component: zammad-nginx
+spec:
+  replicas: {{ .Values.zammadConfig.nginx.replicas }}
+  selector:
+    matchLabels:
+      {{- include "zammad.selectorLabels" . | nindent 6 }}
+  template:
+    metadata:
+      {{- with .Values.podAnnotations }}
+      annotations:
+        {{- toYaml . | nindent 8 }}
+      {{- end }}
+      labels:
+        {{- include "zammad.labels" . | nindent 8 }}
+        app.kubernetes.io/component: zammad-nginx
+    spec:
+      {{- include "zammad.podSpec.deployment" . | nindent 6 }}
+      containers:
+        {{- with .Values.zammadConfig.nginx.sidecars }}
+        {{- toYaml . | nindent 8}}
+        {{- end }}
+        - name: {{ .Chart.Name }}-nginx
+          {{- include "zammad.containerSpec" (merge (dict "containerConfig" .Values.zammadConfig.nginx) .) | nindent 10 }}
+          command:
+            - /usr/sbin/nginx
+            - -g
+            - 'daemon off;'
+          env:
+            {{- include "zammad.env" . | nindent 12 }}
+            {{- include "zammad.env.failOnPendingMigrations" . | nindent 12 }}
+          ports:
+            - name: http
+              containerPort: 8080
+          volumeMounts:
+            {{- include "zammad.volumeMounts" . | nindent 12 }}
+            - name: {{ include "zammad.fullname" . }}-nginx
+              mountPath: /etc/nginx/nginx.conf
+              subPath: nginx.conf
+              readOnly: true
+            - name: {{ include "zammad.fullname" . }}-nginx
+              mountPath: /etc/nginx/sites-enabled/default
+              subPath: default
+              readOnly: true
+            - name: {{ include "zammad.fullname" . }}-tmp
+              mountPath: /var/log/nginx
+      volumes:
+        {{- include "zammad.volumes" . | nindent 8 }}
+        - name: {{ template "zammad.fullname" . }}-init
+          configMap:
+            name: {{ template "zammad.fullname" . }}-init
+            defaultMode: 0755
+        - name: {{ template "zammad.fullname" . }}-nginx
+          configMap:
+            name: {{ template "zammad.fullname" . }}-nginx
diff --git a/zammad/templates/deployment-railsserver.yaml b/zammad/templates/deployment-railsserver.yaml
new file mode 100644
index 00000000..11fa55a8
--- /dev/null
+++ b/zammad/templates/deployment-railsserver.yaml
@@ -0,0 +1,49 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: {{ template "zammad.fullname" . }}-railsserver
+  labels:
+    {{- include "zammad.labels" . | nindent 4 }}
+    app.kubernetes.io/component: zammad-railsserver
+spec:
+  replicas: {{ .Values.zammadConfig.railsserver.replicas }}
+  selector:
+    matchLabels:
+      {{- include "zammad.selectorLabels" . | nindent 6 }}
+  template:
+    metadata:
+      {{- with .Values.podAnnotations }}
+      annotations:
+        {{- toYaml . | nindent 8 }}
+      {{- end }}
+      labels:
+        {{- include "zammad.labels" . | nindent 8 }}
+        app.kubernetes.io/component: zammad-railsserver
+    spec:
+      {{- include "zammad.podSpec.deployment" . | nindent 6 }}
+      containers:
+        {{- with .Values.zammadConfig.railsserver.sidecars }}
+        {{- toYaml . | nindent 8}}
+        {{- end }}
+        - name: {{ .Chart.Name }}-railsserver
+          {{- include "zammad.containerSpec" (merge (dict "containerConfig" .Values.zammadConfig.railsserver) .) | nindent 10 }}
+          command:
+            - "bundle"
+            - "exec"
+            - "puma"
+            - "-b"
+            - "tcp://[::]:3000"
+            - "-w"
+            - "{{ .Values.zammadConfig.railsserver.webConcurrency }}"
+            - "-e"
+            - "production"
+          env:
+            {{- include "zammad.env" . | nindent 12 }}
+            {{- include "zammad.env.failOnPendingMigrations" . | nindent 12 }}
+          ports:
+            - name: railsserver
+              containerPort: 3000
+          volumeMounts:
+            {{- include "zammad.volumeMounts" . | nindent 12 }}
+      volumes:
+        {{- include "zammad.volumes" . | nindent 8 }}
diff --git a/zammad/templates/deployment-scheduler.yaml b/zammad/templates/deployment-scheduler.yaml
new file mode 100644
index 00000000..7bc6dfeb
--- /dev/null
+++ b/zammad/templates/deployment-scheduler.yaml
@@ -0,0 +1,41 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: {{ template "zammad.fullname" . }}-scheduler
+  labels:
+    {{- include "zammad.labels" . | nindent 4 }}
+    app.kubernetes.io/component: zammad-scheduler
+spec:
+  replicas: 1 # Not scalable, may only run once per cluster.
+  selector:
+    matchLabels:
+      {{- include "zammad.selectorLabels" . | nindent 6 }}
+  template:
+    metadata:
+      {{- with .Values.podAnnotations }}
+      annotations:
+        {{- toYaml . | nindent 8 }}
+      {{- end }}
+      labels:
+        {{- include "zammad.labels" . | nindent 8 }}
+        app.kubernetes.io/component: zammad-scheduler
+    spec:
+      {{- include "zammad.podSpec.deployment" . | nindent 6 }}
+      containers:
+        {{- with .Values.zammadConfig.scheduler.sidecars }}
+        {{- toYaml . | nindent 8}}
+        {{- end }}
+        - name: {{ .Chart.Name }}-scheduler
+          {{- include "zammad.containerSpec" (merge (dict "containerConfig" .Values.zammadConfig.scheduler) .) | nindent 10 }}
+          command:
+            - "bundle"
+            - "exec"
+            - "script/background-worker.rb"
+            - "start"
+          env:
+            {{- include "zammad.env" . | nindent 12 }}
+            {{- include "zammad.env.failOnPendingMigrations" . | nindent 12 }}
+          volumeMounts:
+            {{- include "zammad.volumeMounts" . | nindent 12 }}
+      volumes:
+        {{- include "zammad.volumes" . | nindent 8 }}
diff --git a/zammad/templates/deployment-websocket.yaml b/zammad/templates/deployment-websocket.yaml
new file mode 100644
index 00000000..deb53636
--- /dev/null
+++ b/zammad/templates/deployment-websocket.yaml
@@ -0,0 +1,48 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: {{ template "zammad.fullname" . }}-websocket
+  labels:
+    {{- include "zammad.labels" . | nindent 4 }}
+    app.kubernetes.io/component: zammad-websocket
+spec:
+  replicas: 1 # Not scalable, may only run once per cluster.
+  selector:
+    matchLabels:
+      {{- include "zammad.selectorLabels" . | nindent 6 }}
+  template:
+    metadata:
+      {{- with .Values.podAnnotations }}
+      annotations:
+        {{- toYaml . | nindent 8 }}
+      {{- end }}
+      labels:
+        {{- include "zammad.labels" . | nindent 8 }}
+        app.kubernetes.io/component: zammad-websocket
+    spec:
+      {{- include "zammad.podSpec.deployment" . | nindent 6 }}
+      containers:
+        {{- with .Values.zammadConfig.websocket.sidecars }}
+        {{- toYaml . | nindent 8}}
+        {{- end }}
+        - name: {{ .Chart.Name }}-websocket
+          {{- include "zammad.containerSpec" (merge (dict "containerConfig" .Values.zammadConfig.websocket) .) | nindent 10 }}
+          command:
+            - "bundle"
+            - "exec"
+            - "script/websocket-server.rb"
+            - "-b"
+            - "0.0.0.0"
+            - "-p"
+            - "6042"
+            - "start"
+          env:
+            {{- include "zammad.env" . | nindent 12 }}
+            {{- include "zammad.env.failOnPendingMigrations" . | nindent 12 }}
+          ports:
+            - name: websocket
+              containerPort: 6042
+          volumeMounts:
+            {{- include "zammad.volumeMounts" . | nindent 12 }}
+      volumes:
+        {{- include "zammad.volumes" . | nindent 8 }}
diff --git a/zammad/templates/ingress.yaml b/zammad/templates/ingress.yaml
index f4b36e3c..6c015c85 100644
--- a/zammad/templates/ingress.yaml
+++ b/zammad/templates/ingress.yaml
@@ -49,11 +49,11 @@ spec:
             backend:
               {{- if semverCompare ">=1.19-0" $.Capabilities.KubeVersion.GitVersion }}
               service:
-                name: {{ $fullName }}
+                name: zammad-nginx
                 port:
                   number: {{ $svcPort }}
               {{- else }}
-              serviceName: {{ $fullName }}
+              serviceName: zammad-nginx
               servicePort: {{ $svcPort }}
               {{- end }}
           {{- end }}
diff --git a/zammad/templates/job-init.yaml b/zammad/templates/job-init.yaml
new file mode 100644
index 00000000..006b324b
--- /dev/null
+++ b/zammad/templates/job-init.yaml
@@ -0,0 +1,93 @@
+apiVersion: batch/v1
+kind: Job
+metadata:
+  name: {{ template "zammad.fullname" . }}-init-{{ uuidv4 }}
+  # Use a different job name on each run to ensure a new job always runs once.
+  # Helm post-install/post-upgrade hooks cannot be used here, because
+  #   helm's --wait flag causes a deadlock: the job waits for all resources to be ready,
+  #   but the pods need the job to work properly.
+  labels:
+    {{- include "zammad.labels" . | nindent 4 }}
+spec:
+  ttlSecondsAfterFinished: 300
+  template:
+    metadata:
+      {{- with .Values.podAnnotations }}
+      annotations:
+        {{- toYaml . | nindent 8 }}
+      {{- end }}
+      labels:
+        {{- include "zammad.labels" . | nindent 8 }}
+    spec:
+      {{- include "zammad.podSpec" . | nindent 6 }}
+      restartPolicy: OnFailure
+      containers:
+        {{- with .Values.initContainers }}
+        {{- toYaml . | nindent 8}}
+        {{- end }}
+        {{- if .Values.zammadConfig.initContainers.volumePermissions.enabled }}
+        - name: zammad-volume-permissions
+          image: "{{ .Values.zammadConfig.initContainers.volumePermissions.image.repository }}:{{ .Values.zammadConfig.initContainers.volumePermissions.image.tag }}"
+          imagePullPolicy: {{ .Values.zammadConfig.initContainers.volumePermissions.image.pullPolicy }}
+          command:
+            {{- .Values.zammadConfig.initContainers.volumePermissions.command | toYaml | nindent 12 }}
+          {{- with .Values.zammadConfig.initContainers.volumePermissions.resources }}
+          resources:
+            {{- toYaml . | nindent 12 }}
+          {{- end }}
+          {{- with .Values.zammadConfig.initContainers.volumePermissions.securityContext }}
+          securityContext:
+            {{- toYaml . | nindent 12 }}
+          {{- end }}
+          volumeMounts:
+            {{- include "zammad.volumeMounts" . | nindent 12 }}
+        {{- end }}
+        - name: postgresql-init
+          {{- include "zammad.containerSpec" (merge (dict "containerConfig" .Values.zammadConfig.initContainers.postgresql) .) | nindent 10 }}
+          env:
+            {{- include "zammad.env" . | nindent 12 }}
+          volumeMounts:
+            {{- include "zammad.volumeMounts" . | nindent 12 }}
+            - name: {{ template "zammad.fullname" . }}-init
+              mountPath: /docker-entrypoint.sh
+              readOnly: true
+              subPath: postgresql-init
+        {{- if .Values.zammadConfig.initContainers.zammad.customInit }}
+        - name: zammad-init
+          {{- include "zammad.containerSpec" (merge (dict "containerConfig" .Values.zammadConfig.initContainers.zammad) .) | nindent 10 }}
+          env:
+            {{- include "zammad.env" . | nindent 12 }}
+            {{- include "zammad.env.failOnPendingMigrations" . | nindent 12 }}
+          volumeMounts:
+            {{- include "zammad.volumeMounts" . | nindent 12 }}
+            - name: {{ template "zammad.fullname" . }}-init
+              mountPath: /docker-entrypoint.sh
+              readOnly: true
+              subPath: zammad-init
+        {{- end }}
+        {{- if .Values.zammadConfig.elasticsearch.initialisation }}
+        - name: elasticsearch-init
+          {{- include "zammad.containerSpec" (merge (dict "containerConfig" .Values.zammadConfig.initContainers.elasticsearch) .) | nindent 10 }}
+          env:
+            {{- include "zammad.env" . | nindent 12 }}
+            {{- include "zammad.env.failOnPendingMigrations" . | nindent 12 }}
+          {{- if or .Values.zammadConfig.elasticsearch.pass .Values.secrets.elasticsearch.useExisting }}
+            - name: ELASTICSEARCH_PASSWORD
+              valueFrom:
+                secretKeyRef:
+                  name: {{ template "zammad.elasticsearchSecretName" . }}
+                  key: {{ .Values.secrets.elasticsearch.secretKey }}
+          {{- end }}
+          volumeMounts:
+            {{- include "zammad.volumeMounts" . | nindent 12 }}
+            - name: {{ template "zammad.fullname" . }}-init
+              mountPath: /docker-entrypoint.sh
+              readOnly: true
+              subPath: elasticsearch-init
+        {{- end }}
+      volumes:
+        {{- include "zammad.volumes" . | nindent 8 }}
+        - name: {{ template "zammad.fullname" . }}-init
+          configMap:
+            name: {{ template "zammad.fullname" . }}-init
+            defaultMode: 0755
\ No newline at end of file
diff --git a/zammad/templates/service.yaml b/zammad/templates/service-nginx.yaml
similarity index 77%
rename from zammad/templates/service.yaml
rename to zammad/templates/service-nginx.yaml
index c1fdc79d..f198db25 100644
--- a/zammad/templates/service.yaml
+++ b/zammad/templates/service-nginx.yaml
@@ -1,7 +1,7 @@
 apiVersion: v1
 kind: Service
 metadata:
-  name: {{ include "zammad.fullname" . }}
+  name: {{ include "zammad.fullname" . }}-nginx
   labels:
     {{- include "zammad.labels" . | nindent 4 }}
 spec:
@@ -13,3 +13,4 @@ spec:
       name: http
   selector:
     {{- include "zammad.selectorLabels" . | nindent 4 }}
+    app.kubernetes.io/component: zammad-nginx
diff --git a/zammad/templates/service-railsserver.yaml b/zammad/templates/service-railsserver.yaml
new file mode 100644
index 00000000..69f32d20
--- /dev/null
+++ b/zammad/templates/service-railsserver.yaml
@@ -0,0 +1,15 @@
+apiVersion: v1
+kind: Service
+metadata:
+  name: {{ include "zammad.fullname" . }}-railsserver
+  labels:
+    {{- include "zammad.labels" . | nindent 4 }}
+spec:
+  ports:
+    - port: 3000
+      targetPort: 3000
+      protocol: TCP
+      name: http
+  selector:
+    {{- include "zammad.selectorLabels" . | nindent 4 }}
+    app.kubernetes.io/component: zammad-railsserver
diff --git a/zammad/templates/service-websocket.yaml b/zammad/templates/service-websocket.yaml
new file mode 100644
index 00000000..df8ffa30
--- /dev/null
+++ b/zammad/templates/service-websocket.yaml
@@ -0,0 +1,15 @@
+apiVersion: v1
+kind: Service
+metadata:
+  name: {{ include "zammad.fullname" . }}-websocket
+  labels:
+    {{- include "zammad.labels" . | nindent 4 }}
+spec:
+  ports:
+    - port: 6042
+      targetPort: 6042
+      protocol: TCP
+      name: http
+  selector:
+    {{- include "zammad.selectorLabels" . | nindent 4 }}
+    app.kubernetes.io/component: zammad-websocket
diff --git a/zammad/templates/statefulset.yaml b/zammad/templates/statefulset.yaml
deleted file mode 100644
index 85b61513..00000000
--- a/zammad/templates/statefulset.yaml
+++ /dev/null
@@ -1,485 +0,0 @@
-apiVersion: apps/v1
-kind: StatefulSet
-metadata:
-  name: {{ template "zammad.fullname" . }}
-  labels:
-    {{- include "zammad.labels" . | nindent 4 }}
-spec:
-  replicas: {{ .Values.replicas }}
-  serviceName: {{ include "zammad.name" . }}
-  selector:
-    matchLabels:
-      {{- include "zammad.selectorLabels" . | nindent 6 }}
-  template:
-    metadata:
-      {{- with .Values.podAnnotations }}
-      annotations:
-        {{- toYaml . | nindent 8 }}
-      {{- end }}
-      labels:
-        {{- include "zammad.labels" . | nindent 8 }}
-    spec:
-      {{- with .Values.image.imagePullSecrets }}
-      imagePullSecrets:
-        {{- toYaml . | nindent 8 }}
-      {{- end }}
-      {{- if .Values.serviceAccount.create }}
-      serviceAccountName: {{ include "zammad.serviceAccountName" . }}
-      {{- end }}
-      {{- with .Values.nodeSelector }}
-      nodeSelector:
-        {{- toYaml . | nindent 8 }}
-      {{- end }}
-      {{- with .Values.affinity }}
-      affinity:
-        {{- toYaml . | nindent 8 }}
-      {{- end }}
-      {{- with .Values.tolerations }}
-      tolerations:
-        {{- toYaml . | nindent 8 }}
-      {{- end }}
-      initContainers:
-        {{- with .Values.initContainers }}
-        {{- toYaml . | nindent 8}}
-        {{- end }}
-        {{- if .Values.zammadConfig.initContainers.volumePermissions.enabled }}
-        - name: zammad-volume-permissions
-          image: "{{ .Values.zammadConfig.initContainers.volumePermissions.image.repository }}:{{ .Values.zammadConfig.initContainers.volumePermissions.image.tag }}"
-          imagePullPolicy: {{ .Values.zammadConfig.initContainers.volumePermissions.image.pullPolicy }}
-          command:
-            - /bin/sh
-            - -cx
-            - |
-              chmod 770 /opt/zammad/tmp
-          {{- with .Values.zammadConfig.initContainers.volumePermissions.resources }}
-          resources:
-            {{- toYaml . | nindent 12 }}
-          {{- end }}
-          {{- with .Values.zammadConfig.initContainers.volumePermissions.securityContext }}
-          securityContext:
-            {{- toYaml . | nindent 12 }}
-          {{- end }}
-          volumeMounts:
-            - name: {{ template "zammad.fullname" . }}-tmp
-              mountPath: /opt/zammad/tmp
-        {{- end }}
-        - name: postgresql-init
-          image: "{{ .Values.image.repository }}:{{ .Values.image.tag | default .Chart.AppVersion }}"
-          imagePullPolicy: {{ .Values.image.pullPolicy }}
-          env:
-          {{- if or .Values.zammadConfig.redis.pass .Values.secrets.redis.useExisting }}
-            - name: REDIS_PASSWORD
-              valueFrom:
-                secretKeyRef:
-                  name: {{ template "zammad.redisSecretName" . }}
-                  key: {{ .Values.secrets.redis.secretKey }}
-          {{- end }}
-            - name: MEMCACHE_SERVERS
-              value: "{{ if .Values.zammadConfig.memcached.enabled }}{{ .Release.Name }}-memcached{{ else }}{{ .Values.zammadConfig.memcached.host }}{{ end }}:{{ .Values.zammadConfig.memcached.port }}"
-            - name: REDIS_URL
-              value: "redis://:$(REDIS_PASSWORD)@{{ if .Values.zammadConfig.redis.enabled }}{{ .Release.Name }}-redis-master{{ else }}{{ .Values.zammadConfig.redis.host }}{{ end }}:{{ .Values.zammadConfig.redis.port }}"
-            - name: POSTGRESQL_PASS
-              valueFrom:
-                secretKeyRef:
-                  name: {{ template "zammad.postgresqlSecretName" . }}
-                  key: {{ .Values.secrets.postgresql.secretKey }}
-            - name: DATABASE_URL
-              value: "postgres://{{ .Values.zammadConfig.postgresql.user }}:$(POSTGRESQL_PASS)@{{ if .Values.zammadConfig.postgresql.enabled }}{{ .Release.Name }}-postgresql{{ else }}{{ .Values.zammadConfig.postgresql.host }}{{ end }}:{{ .Values.zammadConfig.postgresql.port }}/{{ .Values.zammadConfig.postgresql.db }}?{{ .Values.zammadConfig.postgresql.options }}"
-            {{ include "zammad.env.S3_URL" . | nindent 12 }}
-          {{- if .Values.extraEnv }}
-            {{- toYaml .Values.extraEnv | nindent 12 }}
-          {{- end }}
-          {{- with .Values.zammadConfig.initContainers.postgresql.resources }}
-          resources:
-            {{- toYaml . | nindent 12 }}
-          {{- end }}
-          {{- with .Values.zammadConfig.initContainers.postgresql.securityContext }}
-          securityContext:
-            {{- toYaml . | nindent 12 }}
-          {{- end }}
-          volumeMounts:
-            - name: {{ template "zammad.fullname" . }}-init
-              mountPath: /docker-entrypoint.sh
-              readOnly: true
-              subPath: postgresql-init
-            - name: {{ template "zammad.fullname" . }}-tmp
-              mountPath: /opt/zammad/tmp
-            - name: {{ template "zammad.fullname" . }}-var
-              mountPath: /opt/zammad/storage
-      {{- if .Values.zammadConfig.initContainers.zammad.customInit }}
-        - name: zammad-init
-          image: "{{ .Values.image.repository }}:{{ .Values.image.tag | default .Chart.AppVersion }}"
-          imagePullPolicy: {{ .Values.image.pullPolicy }}
-          env:
-          {{- if or .Values.zammadConfig.redis.pass .Values.secrets.redis.useExisting }}
-            - name: REDIS_PASSWORD
-              valueFrom:
-                secretKeyRef:
-                  name: {{ template "zammad.redisSecretName" . }}
-                  key: {{ .Values.secrets.redis.secretKey }}
-          {{- end }}
-            - name: MEMCACHE_SERVERS
-              value: "{{ if .Values.zammadConfig.memcached.enabled }}{{ .Release.Name }}-memcached{{ else }}{{ .Values.zammadConfig.memcached.host }}{{ end }}:{{ .Values.zammadConfig.memcached.port }}"
-            - name: REDIS_URL
-              value: "redis://:$(REDIS_PASSWORD)@{{ if .Values.zammadConfig.redis.enabled }}{{ .Release.Name }}-redis-master{{ else }}{{ .Values.zammadConfig.redis.host }}{{ end }}:{{ .Values.zammadConfig.redis.port }}"
-            - name: POSTGRESQL_PASS
-              valueFrom:
-                secretKeyRef:
-                  name: {{ template "zammad.postgresqlSecretName" . }}
-                  key: {{ .Values.secrets.postgresql.secretKey }}
-            - name: DATABASE_URL
-              value: "postgres://{{ .Values.zammadConfig.postgresql.user }}:$(POSTGRESQL_PASS)@{{ if .Values.zammadConfig.postgresql.enabled }}{{ .Release.Name }}-postgresql{{ else }}{{ .Values.zammadConfig.postgresql.host }}{{ end }}:{{ .Values.zammadConfig.postgresql.port }}/{{ .Values.zammadConfig.postgresql.db }}?{{ .Values.zammadConfig.postgresql.options }}"
-            {{ include "zammad.env.S3_URL" . | nindent 12 }}
-          {{- if .Values.extraEnv }}
-          {{- toYaml .Values.extraEnv | nindent 12 }}
-          {{- end }}
-          {{- with .Values.zammadConfig.initContainers.zammad.resources }}
-          resources:
-            {{- toYaml . | nindent 12 }}
-          {{- end }}
-          {{- with .Values.zammadConfig.initContainers.zammad.securityContext }}
-          securityContext:
-            {{- toYaml . | nindent 12 }}
-          {{- end }}
-          volumeMounts:
-            - name: {{ template "zammad.fullname" . }}-init
-              mountPath: /docker-entrypoint.sh
-              readOnly: true
-              subPath: zammad-init
-            - name: {{ template "zammad.fullname" . }}-tmp
-              mountPath: /opt/zammad/tmp
-            - name: {{ template "zammad.fullname" . }}-var
-              mountPath: /opt/zammad/storage
-        {{- end }}
-        {{- if .Values.zammadConfig.elasticsearch.initialisation }}
-        - name: elasticsearch-init
-          image: "{{ .Values.image.repository }}:{{ .Values.image.tag | default .Chart.AppVersion }}"
-          imagePullPolicy: {{ .Values.image.pullPolicy }}
-          env:
-          {{- if or .Values.zammadConfig.redis.pass .Values.secrets.redis.useExisting }}
-            - name: REDIS_PASSWORD
-              valueFrom:
-                secretKeyRef:
-                  name: {{ template "zammad.redisSecretName" . }}
-                  key: {{ .Values.secrets.redis.secretKey }}
-          {{- end }}
-            - name: MEMCACHE_SERVERS
-              value: "{{ if .Values.zammadConfig.memcached.enabled }}{{ .Release.Name }}-memcached{{ else }}{{ .Values.zammadConfig.memcached.host }}{{ end }}:{{ .Values.zammadConfig.memcached.port }}"
-            - name: REDIS_URL
-              value: "redis://:$(REDIS_PASSWORD)@{{ if .Values.zammadConfig.redis.enabled }}{{ .Release.Name }}-redis-master{{ else }}{{ .Values.zammadConfig.redis.host }}{{ end }}:{{ .Values.zammadConfig.redis.port }}"
-            - name: POSTGRESQL_PASS
-              valueFrom:
-                secretKeyRef:
-                  name: {{ template "zammad.postgresqlSecretName" . }}
-                  key: {{ .Values.secrets.postgresql.secretKey }}
-            - name: DATABASE_URL
-              value: "postgres://{{ .Values.zammadConfig.postgresql.user }}:$(POSTGRESQL_PASS)@{{ if .Values.zammadConfig.postgresql.enabled }}{{ .Release.Name }}-postgresql{{ else }}{{ .Values.zammadConfig.postgresql.host }}{{ end }}:{{ .Values.zammadConfig.postgresql.port }}/{{ .Values.zammadConfig.postgresql.db }}?{{ .Values.zammadConfig.postgresql.options }}"
-            {{ include "zammad.env.S3_URL" . | nindent 12 }}
-          {{- if or .Values.zammadConfig.elasticsearch.pass .Values.secrets.elasticsearch.useExisting }}
-            - name: ELASTICSEARCH_PASSWORD
-              valueFrom:
-                secretKeyRef:
-                  name: {{ template "zammad.elasticsearchSecretName" . }}
-                  key: {{ .Values.secrets.elasticsearch.secretKey }}
-          {{- end }}
-          {{- if .Values.extraEnv }}
-          {{- toYaml .Values.extraEnv | nindent 12 }}
-          {{- end }}
-          {{- with .Values.zammadConfig.initContainers.elasticsearch.resources }}
-          resources:
-            {{- toYaml . | nindent 12 }}
-          {{- end }}
-          {{- with .Values.zammadConfig.initContainers.elasticsearch.securityContext}}
-          securityContext:
-            {{- toYaml . | nindent 12 }}
-          {{- end }}
-          volumeMounts:
-          - name: {{ template "zammad.fullname" . }}-init
-            mountPath: /docker-entrypoint.sh
-            readOnly: true
-            subPath: elasticsearch-init
-          - name: {{ template "zammad.fullname" . }}-tmp
-            mountPath: /opt/zammad/tmp
-          - name: {{ template "zammad.fullname" . }}-var
-            mountPath: /opt/zammad/storage
-        {{- end }}
-      containers:
-        {{- with .Values.sidecars }}
-        {{- toYaml . | nindent 8}}
-        {{- end }}
-        - name: {{ .Chart.Name }}-nginx
-          image: "{{ .Values.image.repository }}:{{ .Values.image.tag | default .Chart.AppVersion }}"
-          imagePullPolicy: {{ .Values.image.pullPolicy }}
-          command:
-            - /usr/sbin/nginx
-            - -g
-            - 'daemon off;'
-          env:
-          {{- if .Values.extraEnv }}
-            {{- toYaml .Values.extraEnv | nindent 12 }}
-          {{- end }}
-          ports:
-            - name: http
-              containerPort: 8080
-          {{- with .Values.zammadConfig.nginx.livenessProbe }}
-          livenessProbe:
-            {{- toYaml . | nindent 12 }}
-          {{- end }}
-          {{- with .Values.zammadConfig.nginx.readinessProbe }}
-          readinessProbe:
-            {{- toYaml . | nindent 12 }}
-          {{- end }}
-          {{- with .Values.zammadConfig.nginx.resources }}
-          resources:
-            {{- toYaml . | nindent 12 }}
-          {{- end }}
-          {{- with .Values.zammadConfig.nginx.securityContext }}
-          securityContext:
-            {{- toYaml . | nindent 12 }}
-          {{- end }}
-          volumeMounts:
-            - name: {{ include "zammad.fullname" . }}-nginx
-              mountPath: /etc/nginx/nginx.conf
-              subPath: nginx.conf
-              readOnly: true
-            - name: {{ include "zammad.fullname" . }}-nginx
-              mountPath: /etc/nginx/sites-enabled/default
-              subPath: default
-              readOnly: true
-            - name: {{ template "zammad.fullname" . }}-tmp
-              mountPath: /tmp
-            - name: {{ include "zammad.fullname" . }}-tmp
-              mountPath: /var/log/nginx
-            - name: {{ template "zammad.fullname" . }}-var
-              mountPath: /opt/zammad/storage
-        - name: {{ .Chart.Name }}-railsserver
-          image: "{{ .Values.image.repository }}:{{ .Values.image.tag | default .Chart.AppVersion }}"
-          imagePullPolicy: {{ .Values.image.pullPolicy }}
-          command:
-            - "bundle"
-            - "exec"
-            - "puma"
-            - "-b"
-            - "tcp://[::]:3000"
-            - "-w"
-            - "{{ .Values.zammadConfig.railsserver.webConcurrency }}"
-            - "-e"
-            - "production"
-          env:
-            - name: TMP
-              value: {{ .Values.zammadConfig.railsserver.tmpdir }}
-          {{- if .Values.autoWizard.enabled }}
-            - name: AUTOWIZARD_RELATIVE_PATH
-              value: tmp/auto_wizard/auto_wizard.json
-          {{- end }}
-          {{- if or .Values.zammadConfig.redis.pass .Values.secrets.redis.useExisting }}
-            - name: REDIS_PASSWORD
-              valueFrom:
-                secretKeyRef:
-                  name: {{ template "zammad.redisSecretName" . }}
-                  key: {{ .Values.secrets.redis.secretKey }}
-          {{- end }}
-            - name: MEMCACHE_SERVERS
-              value: "{{ if .Values.zammadConfig.memcached.enabled }}{{ .Release.Name }}-memcached{{ else }}{{ .Values.zammadConfig.memcached.host }}{{ end }}:{{ .Values.zammadConfig.memcached.port }}"
-            - name: RAILS_TRUSTED_PROXIES
-              value: "{{ .Values.zammadConfig.railsserver.trustedProxies }}"
-            - name: REDIS_URL
-              value: "redis://:$(REDIS_PASSWORD)@{{ if .Values.zammadConfig.redis.enabled }}{{ .Release.Name }}-redis-master{{ else }}{{ .Values.zammadConfig.redis.host }}{{ end }}:{{ .Values.zammadConfig.redis.port }}"
-            - name: POSTGRESQL_PASS
-              valueFrom:
-                secretKeyRef:
-                  name: {{ template "zammad.postgresqlSecretName" . }}
-                  key: {{ .Values.secrets.postgresql.secretKey }}
-            - name: DATABASE_URL
-              value: "postgres://{{ .Values.zammadConfig.postgresql.user }}:$(POSTGRESQL_PASS)@{{ if .Values.zammadConfig.postgresql.enabled }}{{ .Release.Name }}-postgresql{{ else }}{{ .Values.zammadConfig.postgresql.host }}{{ end }}:{{ .Values.zammadConfig.postgresql.port }}/{{ .Values.zammadConfig.postgresql.db }}?{{ .Values.zammadConfig.postgresql.options }}"
-            {{ include "zammad.env.S3_URL" . | nindent 12 }}
-          {{- if .Values.extraEnv }}
-          {{- toYaml .Values.extraEnv | nindent 12 }}
-          {{- end }}
-          ports:
-            - name: railsserver
-              containerPort: 3000
-          {{- with .Values.zammadConfig.railsserver.livenessProbe }}
-          livenessProbe:
-            {{- toYaml . | nindent 12 }}
-          {{- end }}
-          {{- with .Values.zammadConfig.railsserver.readinessProbe }}
-          readinessProbe:
-            {{- toYaml . | nindent 12 }}
-          {{- end }}
-          {{- with .Values.zammadConfig.railsserver.resources }}
-          resources:
-            {{- toYaml . | nindent 12 }}
-          {{- end }}
-          {{- with .Values.zammadConfig.railsserver.securityContext }}
-          securityContext:
-            {{- toYaml . | nindent 12 }}
-          {{- end }}
-          volumeMounts:
-            - name: {{ template "zammad.fullname" . }}-tmp
-              mountPath: /tmp
-            - name: {{ template "zammad.fullname" . }}-tmp
-              mountPath: /opt/zammad/tmp
-            - name: {{ template "zammad.fullname" . }}-var
-              mountPath: /opt/zammad/storage
-          {{- if .Values.autoWizard.enabled }}
-            - name: autowizard
-              mountPath: "/opt/zammad/tmp/auto_wizard"
-          {{- end }}
-        - name: {{ .Chart.Name }}-scheduler
-          image: "{{ .Values.image.repository }}:{{ .Values.image.tag | default .Chart.AppVersion }}"
-          imagePullPolicy: {{ .Values.image.pullPolicy }}
-          command:
-            - "bundle"
-            - "exec"
-            - "script/background-worker.rb"
-            - "start"
-          env:
-          {{- if or .Values.zammadConfig.redis.pass .Values.secrets.redis.useExisting }}
-            - name: REDIS_PASSWORD
-              valueFrom:
-                secretKeyRef:
-                  name: {{ template "zammad.redisSecretName" . }}
-                  key: {{ .Values.secrets.redis.secretKey }}
-          {{- end }}
-            - name: MEMCACHE_SERVERS
-              value: "{{ if .Values.zammadConfig.memcached.enabled }}{{ .Release.Name }}-memcached{{ else }}{{ .Values.zammadConfig.memcached.host }}{{ end }}:{{ .Values.zammadConfig.memcached.port }}"
-            - name: REDIS_URL
-              value: "redis://:$(REDIS_PASSWORD)@{{ if .Values.zammadConfig.redis.enabled }}{{ .Release.Name }}-redis-master{{ else }}{{ .Values.zammadConfig.redis.host }}{{ end }}:{{ .Values.zammadConfig.redis.port }}"
-            - name: POSTGRESQL_PASS
-              valueFrom:
-                secretKeyRef:
-                  name: {{ template "zammad.postgresqlSecretName" . }}
-                  key: {{ .Values.secrets.postgresql.secretKey }}
-            - name: DATABASE_URL
-              value: "postgres://{{ .Values.zammadConfig.postgresql.user }}:$(POSTGRESQL_PASS)@{{ if .Values.zammadConfig.postgresql.enabled }}{{ .Release.Name }}-postgresql{{ else }}{{ .Values.zammadConfig.postgresql.host }}{{ end }}:{{ .Values.zammadConfig.postgresql.port }}/{{ .Values.zammadConfig.postgresql.db }}?{{ .Values.zammadConfig.postgresql.options }}"
-            {{ include "zammad.env.S3_URL" . | nindent 12 }}
-          {{- if .Values.extraEnv }}
-          {{- toYaml .Values.extraEnv | nindent 12 }}
-          {{- end }}
-          {{- with .Values.zammadConfig.scheduler.resources }}
-          resources:
-            {{- toYaml . | nindent 12 }}
-          {{- end }}
-          {{- with .Values.zammadConfig.scheduler.securityContext }}
-          securityContext:
-            {{- toYaml . | nindent 12 }}
-          {{- end }}
-          volumeMounts:
-            - name: {{ template "zammad.fullname" . }}-tmp
-              mountPath: /tmp
-            - name: {{ template "zammad.fullname" . }}-tmp
-              mountPath: /opt/zammad/tmp
-            - name: {{ template "zammad.fullname" . }}-var
-              mountPath: /opt/zammad/storage
-        - name: {{ .Chart.Name }}-websocket
-          image: "{{ .Values.image.repository }}:{{ .Values.image.tag | default .Chart.AppVersion }}"
-          imagePullPolicy: {{ .Values.image.pullPolicy }}
-          command:
-            - "bundle"
-            - "exec"
-            - "script/websocket-server.rb"
-            - "-b"
-            - "0.0.0.0"
-            - "-p"
-            - "6042"
-            - "start"
-          env:
-          {{- if or .Values.zammadConfig.redis.pass .Values.secrets.redis.useExisting }}
-            - name: REDIS_PASSWORD
-              valueFrom:
-                secretKeyRef:
-                  name: {{ template "zammad.redisSecretName" . }}
-                  key: {{ .Values.secrets.redis.secretKey }}
-          {{- end }}
-            - name: MEMCACHE_SERVERS
-              value: "{{ if .Values.zammadConfig.memcached.enabled }}{{ .Release.Name }}-memcached{{ else }}{{ .Values.zammadConfig.memcached.host }}{{ end }}:{{ .Values.zammadConfig.memcached.port }}"
-            - name: REDIS_URL
-              value: "redis://:$(REDIS_PASSWORD)@{{ if .Values.zammadConfig.redis.enabled }}{{ .Release.Name }}-redis-master{{ else }}{{ .Values.zammadConfig.redis.host }}{{ end }}:{{ .Values.zammadConfig.redis.port }}"
-            - name: POSTGRESQL_PASS
-              valueFrom:
-                secretKeyRef:
-                  name: {{ template "zammad.postgresqlSecretName" . }}
-                  key: {{ .Values.secrets.postgresql.secretKey }}
-            - name: DATABASE_URL
-              value: "postgres://{{ .Values.zammadConfig.postgresql.user }}:$(POSTGRESQL_PASS)@{{ if .Values.zammadConfig.postgresql.enabled }}{{ .Release.Name }}-postgresql{{ else }}{{ .Values.zammadConfig.postgresql.host }}{{ end }}:{{ .Values.zammadConfig.postgresql.port }}/{{ .Values.zammadConfig.postgresql.db }}?{{ .Values.zammadConfig.postgresql.options }}"
-            {{ include "zammad.env.S3_URL" . | nindent 12 }}
-          {{- if .Values.extraEnv }}
-          {{- toYaml .Values.extraEnv | nindent 12 }}
-          {{- end }}
-          ports:
-            - name: websocket
-              containerPort: 6042
-          {{- with .Values.zammadConfig.websocket.livenessProbe }}
-          livenessProbe:
-            {{- toYaml . | nindent 12 }}
-          {{- end }}
-          {{- with .Values.zammadConfig.websocket.readinessProbe }}
-          readinessProbe:
-            {{- toYaml . | nindent 12 }}
-          {{- end }}
-          {{- with .Values.zammadConfig.websocket.resources }}
-          resources:
-            {{- toYaml . | nindent 12 }}
-          {{- end }}
-          {{- with .Values.zammadConfig.websocket.securityContext }}
-          securityContext:
-            {{- toYaml . | nindent 12 }}
-          {{- end }}
-          volumeMounts:
-            - name: {{ template "zammad.fullname" . }}-tmp
-              mountPath: /tmp
-            - name: {{ template "zammad.fullname" . }}-tmp
-              mountPath: /opt/zammad/tmp
-            - name: {{ template "zammad.fullname" . }}-var
-              mountPath: /opt/zammad/storage
-      {{- with .Values.securityContext }}
-      securityContext:
-        {{- toYaml . | nindent 8 }}
-      {{- end }}
-      volumes:
-      {{- if .Values.autoWizard.enabled }}
-        - name: autowizard
-          secret:
-            secretName: {{ template "zammad.autowizardSecretName" . }}
-            items:
-            - key: {{ .Values.secrets.autowizard.secretKey }}
-              path: auto_wizard.json
-      {{- end }}
-        - name: {{ template "zammad.fullname" . }}-init
-          configMap:
-            name: {{ template "zammad.fullname" . }}-init
-            defaultMode: 0755
-        - name: {{ template "zammad.fullname" . }}-nginx
-          configMap:
-            name: {{ template "zammad.fullname" . }}-nginx
-        - name: {{ include "zammad.fullname" . }}-tmp
-          {{- toYaml .Values.zammadConfig.tmpDirVolume | nindent 10 }}
-  {{- if and .Values.persistence.enabled .Values.persistence.existingClaim }}
-        - name: {{ template "zammad.fullname" . }}-var
-          persistentVolumeClaim:
-            claimName: {{ .Values.persistence.existingClaim | default (include "zammad.fullname" .) }}
-  {{- else if not .Values.persistence.enabled }}
-        - name: {{ template "zammad.fullname" . }}-var
-          emptyDir:
-            sizeLimit: {{ .Values.persistence.size | quote }}
-  {{- else if and .Values.persistence.enabled (not .Values.persistence.existingClaim) }}
-  volumeClaimTemplates:
-    - metadata:
-        name: {{ template "zammad.fullname" . }}-var
-      spec:
-        accessModes:
-        {{- range .Values.persistence.accessModes }}
-          - {{ . | quote }}
-        {{- end }}
-        resources:
-          requests:
-            storage: {{ .Values.persistence.size | quote }}
-        {{- if .Values.persistence.storageClass }}
-        {{- if (eq "-" .Values.persistence.storageClass) }}
-        storageClassName: ""
-        {{- else }}
-        storageClassName: "{{ .Values.persistence.storageClass }}"
-        {{- end }}
-        {{- end }}
-  {{- end }}
diff --git a/zammad/templates/tests/test-connection.yaml b/zammad/templates/tests/test-connection.yaml
index e75b40c5..9e3c32a9 100644
--- a/zammad/templates/tests/test-connection.yaml
+++ b/zammad/templates/tests/test-connection.yaml
@@ -11,5 +11,5 @@ spec:
     - name: wget
       image: busybox
       command: ['wget']
-      args: ['{{ include "zammad.fullname" . }}:{{ .Values.service.port }}']
+      args: ['{{ include "zammad.fullname" . }}-nginx:{{ .Values.service.port }}']
   restartPolicy: Never
diff --git a/zammad/values.yaml b/zammad/values.yaml
index f412338b..3692a900 100644
--- a/zammad/values.yaml
+++ b/zammad/values.yaml
@@ -92,6 +92,7 @@ zammadConfig:
     # externalS3Url: https://user:pw@external-minio-service/bucket
 
   nginx:
+    replicas: 1
     trustedProxies: []
     extraHeaders: []
       # - 'HeaderName "Header Value"'
@@ -129,6 +130,8 @@ zammadConfig:
           - ALL
       readOnlyRootFilesystem: true
       privileged: false
+    # can be used to add additional containers / sidecars
+    sidecars: []
 
   postgresql:
     # enable/disable postgresql chart dependency
@@ -146,6 +149,7 @@ zammadConfig:
     options: "pool=50"
 
   railsserver:
+    replicas: 1
     livenessProbe:
       tcpSocket:
         port: 3000
@@ -177,8 +181,11 @@ zammadConfig:
           - ALL
       readOnlyRootFilesystem: true
       privileged: false
+    # can be used to add additional containers / sidecars
+    sidecars: []
     trustedProxies: "['127.0.0.1', '::1']"
     webConcurrency: 0
+    # tmpdir will be used by all Zammad/Rails containers
     tmpdir: "/opt/zammad/tmp"
 
   redis:
@@ -204,6 +211,19 @@ zammadConfig:
           - ALL
       readOnlyRootFilesystem: true
       privileged: false
+    # can be used to add additional containers / sidecars
+    sidecars: []
+
+  storageVolume:
+    # Enable this for 'File' based storage in Zammad. You must provide an externally managed 'extistingClaim'
+    #   with 'ReadWriteMany' permisssion in this case.
+    enabled: false
+    ##
+    ## A manually managed Persistent Volume and Claim
+    ## If defined, PVC must be created manually before volume will be bound
+    ## The value is evaluated as a template, so, for example, the name can depend on .Release or .Chart
+    ##
+    # existingClaim:
 
   tmpDirVolume:
     emptyDir:
@@ -243,6 +263,8 @@ zammadConfig:
           - ALL
       readOnlyRootFilesystem: true
       privileged: false
+    # can be used to add additional containers / sidecars
+    sidecars: []
 
   initContainers:
     elasticsearch:
@@ -275,12 +297,18 @@ zammadConfig:
             - ALL
         readOnlyRootFilesystem: true
         privileged: false
+    # volumePermissions will be used by all Zammad Pods
     volumePermissions:
       enabled: true
       image:
         repository: alpine
         tag: "3.18.4"
         pullPolicy: IfNotPresent
+      command:
+        - /bin/sh
+        - -cx
+        - |
+          chmod 770 /opt/zammad/tmp
       resources: {}
         # requests:
         #   cpu: 100m
@@ -360,24 +388,6 @@ autoWizard:
 podAnnotations: {}
   # my-annotation: "value"
 
-# Configuration for persistence
-persistence:
-  enabled: true
-  ## A manually managed Persistent Volume and Claim
-  ## If defined, PVC must be created manually before volume will be bound
-  ## The value is evaluated as a template, so, for example, the name can depend on .Release or .Chart
-  ##
-  # existingClaim:
-  accessModes:
-    - ReadWriteOnce
-  storageClass: ""
-  size: 5Gi
-  annotations: {}
-
-# running zammad with more than 1 replica will need a ReadWriteMany storage volume!
-# https://kubernetes.io/docs/concepts/storage/persistent-volumes/#access-modes
-replicas: 1
-
 nodeSelector: {}
 tolerations: []
 affinity: {}
@@ -411,26 +421,6 @@ initContainers: []
   #     - name: help-zammad
   #       mountPath: /opt/zammad
 
-# can be used to add additional containers / sidecars
-sidecars: []
-  # - name: s3-backup
-  #   image: some-aws-s3-backup:latest
-  #   env:
-  #     - name: AWS_DEFAULT_REGION
-  #       value: "eu-central-1"
-  #     - name: AWS_ACCESS_KEY_ID
-  #       value: "xxxxxxxxxxxx"
-  #     - name: AWS_SECRET_ACCESS_KEY
-  #       value: "xxxxxxxxxxxx"
-  #     - name: SYNC_DIR
-  #       value: "/opt/zammad"
-  #     - name: AWS_SYNC_BUCKET
-  #       value: "some-backup-bucket"
-  #     - name: AWS_SYNC_SCHEDULE
-  #       value: "0 * * * *"
-  #   volumeMounts:
-  #     - name: help-zammad
-  #       mountPath: /opt/zammad
 
 # dependency charts config