DM-43082: Prompt Processing Kafka for Fan Out

applications/prompt-processing-kafka/Chart.yaml

@@ -0,0 +1,15 @@
+apiVersion: v2
+name: prompt-processing-kafka
+version: 1.0.0
+description: "Kafka environment for prompt processing fan out events and bucket notifications"
+sources:
+  - "https://github.com/lsst-dm/prompt_processing"
+  - "https://github.com/lsst-dm/next_visit_fan_out"
+appVersion: 0.1.0
+
+dependencies:
+  - name: strimzi-kafka
+    version: 1.0.0
+  - name: strimzi-registry-operator
+    version: 2.1.0
+    repository: https://lsst-sqre.github.io/charts/

applications/prompt-processing-kafka/README.md

@@ -0,0 +1,86 @@
+# prompt-processing-kafka
+
+Kafka environment for prompt processing fan out events and bucket notifications
+
+## Source Code
+
+* <https://github.com/lsst-dm/prompt_processing>
+* <https://github.com/lsst-dm/next_visit_fan_out>
+
+## Values
+
+| Key | Type | Default | Description |
+|-----|------|---------|-------------|
+| kafdrop.enabled | bool | `false` | Whether Kafdrop is enabled |
+| strimzi-kafka.kafka.listeners.external.enabled | bool | `false` | Whether external listener is enabled |
+| strimzi-kafka.kafka.listeners.noauth.enabled | bool | `true` | Whether internal no authentication listener is enabled |
+| strimzi-kafka.kafka.listeners.plain.enabled | bool | `true` | Whether internal plaintext listener is enabled |
+| strimzi-kafka.kafka.listeners.tls.enabled | bool | `true` | Whether internal TLS listener is enabled |
+| strimzi-registry-operator.clusterName | string | `"prompt-processing-kafka"` | Name of the Strimzi Kafka cluster |
+| strimzi-registry-operator.clusterNamespace | string | `"prompt-processing-kafka"` | Namespace where the Strimzi Kafka cluster is deployed |
+| strimzi-registry-operator.operatorNamespace | string | `"prompt-processing-kafka"` | Namespace where the strimzi-registry-operator is deployed |
+| strimzi-kafka.cluster.monitorLabel | object | `{}` | Site wide label required for gathering Prometheus metrics if they are enabled |
+| strimzi-kafka.cluster.name | string | `"prompt-processing-kafka"` | Name used for the Kafka cluster, and used by Strimzi for many annotations |
+| strimzi-kafka.cruiseControl.enabled | bool | `false` |  |
+| strimzi-kafka.kafka.affinity | object | See `values.yaml` | Affinity for Kafka pod assignment |
+| strimzi-kafka.kafka.config."log.retention.bytes" | string | `"350000000000"` | How much disk space Kafka will ensure is available, set to 70% of the data partition size |
+| strimzi-kafka.kafka.config."log.retention.hours" | int | `48` | Number of days for a topic's data to be retained |
+| strimzi-kafka.kafka.config."message.max.bytes" | int | `10485760` | The largest record batch size allowed by Kafka |
+| strimzi-kafka.kafka.config."offsets.retention.minutes" | int | `2880` | Number of minutes for a consumer group's offsets to be retained |
+| strimzi-kafka.kafka.config."replica.fetch.max.bytes" | int | `10485760` | The number of bytes of messages to attempt to fetch for each partition |
+| strimzi-kafka.kafka.externalListener.bootstrap.annotations | object | `{}` | Annotations that will be added to the Ingress, Route, or Service resource |
+| strimzi-kafka.kafka.externalListener.bootstrap.host | string | Do not configure TLS | Name used for TLS hostname verification |
+| strimzi-kafka.kafka.externalListener.bootstrap.loadBalancerIP | string | Do not request a load balancer IP | Request this load balancer IP. See `values.yaml` for more discussion |
+| strimzi-kafka.kafka.externalListener.brokers | list | `[]` | Brokers configuration. _host_ is used in the brokers' advertised.brokers configuration and for TLS hostname verification.  The format is a list of maps. |
+| strimzi-kafka.kafka.externalListener.tls.certIssuerName | string | `"letsencrypt-dns"` | Name of a ClusterIssuer capable of provisioning a TLS certificate for the broker |
+| strimzi-kafka.kafka.externalListener.tls.enabled | bool | `false` | Whether TLS encryption is enabled |
+| strimzi-kafka.kafka.listeners.external.enabled | bool | `false` | Whether external listener is enabled |
+| strimzi-kafka.kafka.listeners.noauth.enabled | bool | `false` | Whether internal noauth listener is enabled |
+| strimzi-kafka.kafka.listeners.plain.enabled | bool | `false` | Whether internal plaintext listener is enabled |
+| strimzi-kafka.kafka.listeners.tls.enabled | bool | `false` | Whether internal TLS listener is enabled |
+| strimzi-kafka.kafka.metricsConfig.enabled | bool | `false` | Whether metric configuration is enabled |
+| strimzi-kafka.kafka.minInsyncReplicas | int | `2` | The minimum number of in-sync replicas that must be available for the producer to successfully send records Cannot be greater than the number of replicas. |
+| strimzi-kafka.kafka.replicas | int | `3` | Number of Kafka broker replicas to run |
+| strimzi-kafka.kafka.resources | object | See `values.yaml` | Kubernetes requests and limits for the Kafka brokers |
+| strimzi-kafka.kafka.storage.size | string | `"500Gi"` | Size of the backing storage disk for each of the Kafka brokers |
+| strimzi-kafka.kafka.storage.storageClassName | string | `""` | Name of a StorageClass to use when requesting persistent volumes |
+| strimzi-kafka.kafka.tolerations | list | `[]` | Tolerations for Kafka broker pod assignment |
+| strimzi-kafka.kafka.version | string | `"3.8.0"` | Version of Kafka to deploy |
+| strimzi-kafka.kafkaController.enabled | bool | `false` | Enable Kafka Controller |
+| strimzi-kafka.kafkaController.resources | object | See `values.yaml` | Kubernetes requests and limits for the Kafka Controller |
+| strimzi-kafka.kafkaController.storage.size | string | `"20Gi"` | Size of the backing storage disk for each of the Kafka controllers |
+| strimzi-kafka.kafkaController.storage.storageClassName | string | `""` | Name of a StorageClass to use when requesting persistent volumes |
+| strimzi-kafka.kafkaExporter.enableSaramaLogging | bool | `false` | Enable Sarama logging for pod |
+| strimzi-kafka.kafkaExporter.enabled | bool | `false` | Enable Kafka exporter |
+| strimzi-kafka.kafkaExporter.groupRegex | string | `".*"` | Consumer groups to monitor |
+| strimzi-kafka.kafkaExporter.logging | string | `"info"` | Logging level |
+| strimzi-kafka.kafkaExporter.resources | object | See `values.yaml` | Kubernetes requests and limits for the Kafka exporter |
+| strimzi-kafka.kafkaExporter.topicRegex | string | `".*"` | Kafka topics to monitor |
+| strimzi-kafka.kraft.enabled | bool | `false` | Enable KRaft mode for Kafka |
+| strimzi-kafka.registry.ingress.annotations | object | `{}` | Annotations that will be added to the Ingress resource |
+| strimzi-kafka.registry.ingress.enabled | bool | `false` | Whether to enable an ingress for the Schema Registry |
+| strimzi-kafka.registry.ingress.hostname | string | None, must be set if ingress is enabled | Hostname for the Schema Registry |
+| strimzi-kafka.registry.resources | object | See `values.yaml` | Kubernetes requests and limits for the Schema Registry |
+| strimzi-kafka.registry.schemaTopic | string | `"registry-schemas"` | Name of the topic used by the Schema Registry |
+| strimzi-kafka.superusers | list | `["kafka-admin"]` | A list of usernames for users who should have global admin permissions. These users will be created, along with their credentials. |
+| strimzi-kafka.topics | object | `{"hsc":{"enabled":false,"partitions":1,"replicas":1,"retention":3600000},"latiss":{"enabled":false,"partitions":1,"replicas":1,"retention":3600000},"lsstcam":{"enabled":false,"partitions":1,"replicas":1,"retention":3600000},"lsstcomcam":{"enabled":false,"partitions":1,"replicas":1,"retention":3600000},"lsstcomcamsim":{"enabled":false,"partitions":1,"replicas":1,"retention":3600000}}` | Topic configuration.  Enable for supporting certain instruments. |
+| strimzi-kafka.topics.hsc.enabled | bool | `false` | Enable hsc topic |
+| strimzi-kafka.topics.hsc.partitions | int | `1` | Number of partitions on topic |
+| strimzi-kafka.topics.hsc.replicas | int | `1` | Number of replicas |
+| strimzi-kafka.topics.latiss.enabled | bool | `false` | Enable latiss topic |
+| strimzi-kafka.topics.latiss.partitions | int | `1` | Number of partitions on topic |
+| strimzi-kafka.topics.latiss.replicas | int | `1` | Number of replicas |
+| strimzi-kafka.topics.latiss.retention | int | `3600000` | Retention time of events in milliseconds |
+| strimzi-kafka.topics.lsstcam.enabled | bool | `false` | Enable lsstcam topic |
+| strimzi-kafka.topics.lsstcam.partitions | int | `1` | Number of partitions on topic |
+| strimzi-kafka.topics.lsstcam.replicas | int | `1` | Number of replicas |
+| strimzi-kafka.topics.lsstcam.retention | int | `3600000` | Retention time of events in milliseconds |
+| strimzi-kafka.topics.lsstcomcam.enabled | bool | `false` | Enable lsstcomcam topic |
+| strimzi-kafka.topics.lsstcomcam.partitions | int | `1` | Number of partitions on topic |
+| strimzi-kafka.topics.lsstcomcam.replicas | int | `1` | Number of replicas |
+| strimzi-kafka.topics.lsstcomcam.retention | int | `3600000` | Retention time of events in milliseconds |
+| strimzi-kafka.topics.lsstcomcamsim.enabled | bool | `false` | Enable lsstcomcamsim topic |
+| strimzi-kafka.topics.lsstcomcamsim.partitions | int | `1` | Number of partitions on topic |
+| strimzi-kafka.topics.lsstcomcamsim.replicas | int | `1` | Number of replicas |
+| strimzi-kafka.topics.lsstcomcamsim.retention | int | `3600000` | Retention time of events in milliseconds |
+| strimzi-kafka.users.kafdrop.enabled | bool | `false` | Enable user Kafdrop (deployed by parent Prompt Processing Kafka chart). |

applications/prompt-processing-kafka/charts/kafdrop/Chart.yaml

@@ -0,0 +1,7 @@
+apiVersion: v2
+name: kafdrop
+version: 1.0.0
+description: A subchart to deploy the Kafdrop UI for Sasquatch.
+sources:
+  - https://github.com/obsidiandynamics/kafdrop
+appVersion: 3.30.0

applications/prompt-processing-kafka/charts/kafdrop/README.md

@@ -0,0 +1,36 @@
+# kafdrop
+
+A subchart to deploy the Kafdrop UI for Sasquatch.
+
+## Source Code
+
+* <https://github.com/obsidiandynamics/kafdrop>
+
+## Values
+
+| Key | Type | Default | Description |
+|-----|------|---------|-------------|
+| affinity | object | `{}` | Affinity configuration |
+| cmdArgs | string | See `values.yaml` | Command line arguments to Kafdrop |
+| existingSecret | string | Do not use a secret | Existing Kubernetes secrect use to set kafdrop environment variables. Set `SCHEMAREGISTRY_AUTH` for basic auth credentials in the form `<username>:<password>` |
+| host | string | `"localhost"` | The hostname to report for the RMI registry (used for JMX) |
+| image.pullPolicy | string | `"IfNotPresent"` | Image pull policy |
+| image.repository | string | `"obsidiandynamics/kafdrop"` | Kafdrop Docker image repository |
+| image.tag | string | `"4.0.2"` | Kafdrop image version |
+| ingress.annotations | object | `{}` | Additional ingress annotations |
+| ingress.enabled | bool | `false` | Whether to enable the ingress |
+| ingress.hostname | string | None, must be set if ingress is enabled | Ingress hostname |
+| ingress.path | string | `"/kafdrop"` | Ingress path |
+| jmx.port | int | `8686` | Port to use for JMX. If unspecified, JMX will not be exposed. |
+| jvm.opts | string | `""` | JVM options |
+| kafka.broker | string | `""` | Bootstrap list of Kafka host/port pairs |
+| nodeSelector | object | `{}` | Node selector configuration |
+| podAnnotations | object | `{}` | Pod annotations |
+| replicaCount | int | `1` | Number of kafdrop pods to run in the deployment. |
+| resources | object | See `values.yaml` | Kubernetes requests and limits for Kafdrop |
+| schemaregistry | string | `""` | The endpoint of Schema Registry |
+| server.port | int | `9000` | The web server port to listen on |
+| server.servlet.contextPath | string | `"/kafdrop"` | The context path to serve requests on |
+| service.annotations | object | `{}` | Additional annotations to add to the service |
+| service.port | int | `9000` | Service port |
+| tolerations | list | `[]` | Tolerations configuration |

applications/prompt-processing-kafka/charts/kafdrop/templates/NOTES.txt

@@ -0,0 +1,9 @@
+1. Get the application URL by running these commands:
+{{- if .Values.ingress.enabled }}
+{{- range .Values.ingress.hosts }}
+  http{{ if $.Values.ingress.tls }}s{{ end }}://{{ . }}{{ $.Values.ingress.path }}
+{{- end }}
+  export POD_NAME=$(kubectl get pods --namespace {{ .Release.Namespace }} -l "app.kubernetes.io/name={{ include "chart.name" . }},app.kubernetes.io/instance={{ .Release.Name }}" -o jsonpath="{.items[0].metadata.name}")
+  echo "Visit http://127.0.0.1:8080 to use your application"
+  kubectl port-forward $POD_NAME 8080:80
+{{- end }}

applications/prompt-processing-kafka/charts/kafdrop/templates/_helpers.tpl

@@ -0,0 +1,52 @@
+{{/* vim: set filetype=mustache: */}}
+{{/*
+Expand the name of the chart.
+*/}}
+{{- define "chart.name" -}}
+{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" -}}
+{{- end -}}
+
+{{/*
+Create a default fully qualified app name.
+We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec).
+If release name contains chart name it will be used as a full name.
+*/}}
+{{- define "chart.fullname" -}}
+{{- if .Values.fullnameOverride -}}
+{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" -}}
+{{- else -}}
+{{- $name := default .Chart.Name .Values.nameOverride -}}
+{{- if contains $name .Release.Name -}}
+{{- .Release.Name | trunc 63 | trimSuffix "-" -}}
+{{- else -}}
+{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" -}}
+{{- end -}}
+{{- end -}}
+{{- end -}}
+
+{{/*
+Create chart name and version as used by the chart label.
+*/}}
+{{- define "chart.chart" -}}
+{{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" -}}
+{{- end -}}
+
+{{/*
+Common labels
+*/}}
+{{- define "kafdrop.labels" -}}
+helm.sh/chart: {{ include "chart.name" . }}
+{{ include "kafdrop.selectorLabels" . }}
+{{- if .Chart.AppVersion }}
+app.kubernetes.io/version: {{ .Chart.AppVersion | quote }}
+{{- end }}
+app.kubernetes.io/managed-by: {{ .Release.Service }}
+{{- end }}
+
+{{/*
+Selector labels
+*/}}
+{{- define "kafdrop.selectorLabels" -}}
+app.kubernetes.io/name: {{ include "chart.name" . }}
+app.kubernetes.io/instance: {{ .Release.Name }}
+{{- end }}

applications/prompt-processing-kafka/charts/kafdrop/templates/deployment.yaml

@@ -0,0 +1,97 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: {{ include "chart.fullname" . }}
+  labels:
+    {{- include "kafdrop.labels" . | nindent 4 }}
+spec:
+  replicas: {{ .Values.replicaCount }}
+  selector:
+    matchLabels:
+      {{- include "kafdrop.selectorLabels" . | nindent 6 }}
+  template:
+    metadata:
+      labels:
+        {{- include "kafdrop.selectorLabels" . | nindent 8 }}
+      annotations:
+        {{- with .Values.podAnnotations }}
+        {{ toYaml . | indent 8 }}
+        {{- end }}
+    spec:
+      automountServiceAccountToken: false
+      securityContext:
+        runAsNonRoot: true
+        runAsUser: 1000
+        runAsGroup: 1000
+      containers:
+        - name: {{ .Chart.Name }}
+          image: "{{ .Values.image.repository }}:{{ .Values.image.tag }}"
+          imagePullPolicy: {{ .Values.image.pullPolicy }}
+          securityContext:
+            allowPrivilegeEscalation: false
+            capabilities:
+              drop:
+                - ALL
+          {{- if .Values.existingSecret -}}
+          envFrom:
+          - secretRef:
+              name: {{ .Values.existingSecret }}
+          {{- end }}
+          env:
+          - name: ARGOCD_APP_REVISION
+            value: {{ .Values.global.argocd_revision }}
+          - name: KAFKA_BROKERCONNECT
+            value: {{ .Values.kafka.broker | quote }}
+          - name: JVM_OPTS
+            value: {{ .Values.jvm.opts | quote }}
+          - name: HOST
+            value: {{ .Values.host | quote }}
+          - name: JMX_PORT
+            value: {{ .Values.jmx.port | quote }}
+          - name: SERVER_SERVLET_CONTEXTPATH
+            value: {{ .Values.server.servlet.contextPath | trimSuffix "/" | quote }}
+          - name: SERVER_PORT
+            value: {{ .Values.server.port | quote }}
+          - name: CMD_ARGS
+            value: {{ .Values.cmdArgs | quote }}
+          - name: SCHEMAREGISTRY_CONNECT
+            value: {{ .Values.schemaregistry | quote }}
+          - name: KAFKA_PROPERTIES_FILE
+            value: "/tmp/kafka.properties"
+          - name: KAFKA_PROPERTIES
+            valueFrom:
+              secretKeyRef:
+                name: prompt-processing-kafka-secret
+                key: kafdrop-kafka-properties
+          ports:
+            - name: http
+              containerPort: {{ .Values.server.port }}
+              protocol: TCP
+          livenessProbe:
+            httpGet:
+              path: "{{ .Values.server.servlet.contextPath | trimSuffix "/" }}/actuator/health"
+              port: http
+            initialDelaySeconds: 180
+            periodSeconds: 30
+            timeoutSeconds: 10
+          readinessProbe:
+            httpGet:
+              path: "{{ .Values.server.servlet.contextPath | trimSuffix "/" }}/actuator/health"
+              port: http
+            initialDelaySeconds: 20
+            periodSeconds: 5
+            timeoutSeconds: 10
+          resources:
+            {{- toYaml .Values.resources | nindent 12 }}
+      {{- with .Values.nodeSelector }}
+      nodeSelector:
+        {{ toYaml . | indent 8 }}
+      {{- end }}
+      {{- with .Values.affinity }}
+      affinity:
+        {{ toYaml . | indent 8 }}
+      {{- end }}
+      {{- with .Values.tolerations }}
+      tolerations:
+        {{ toYaml . | indent 8 }}
+      {{- end }}

applications/prompt-processing-kafka/charts/kafdrop/templates/ingress.yaml

@@ -0,0 +1,27 @@
+{{- if .Values.ingress.enabled -}}
+{{- $fullName := include "chart.fullname" . -}}
+{{- $ingressPath := .Values.ingress.path -}}
+apiVersion: networking.k8s.io/v1
+kind: Ingress
+metadata:
+  name: {{ $fullName }}
+  labels:
+    {{- include "kafdrop.labels" . | nindent 4 }}
+  annotations:
+    {{- with .Values.ingress.annotations }}
+    {{ toYaml . | indent 4 }}
+    {{- end }}
+spec:
+  ingressClassName: "nginx"
+  rules:
+    - host: {{ .Values.ingress.hostname | quote }}
+      http:
+        paths:
+          - path: {{ $ingressPath }}
+            pathType: Prefix
+            backend:
+              service:
+                name: {{ $fullName }}
+                port:
+                  number: {{ .Values.service.port }}
+{{- end }}

applications/prompt-processing-kafka/charts/kafdrop/templates/service.yaml

@@ -0,0 +1,15 @@
+apiVersion: v1
+kind: Service
+metadata:
+  name: {{ include "chart.fullname" . }}
+  labels:
+    {{- include "kafdrop.labels" . | nindent 4 }}
+spec:
+  type: ClusterIP
+  ports:
+    - port: {{ .Values.service.port }}
+      targetPort: http
+      protocol: TCP
+      name: http
+  selector:
+    {{- include "kafdrop.selectorLabels" . | nindent 4 }}