Merge pull request #10 from esgf2-us/update-chart

Updates helm chart
esgf2-us · Dec 8, 2023 · 2ad0681 · 2ad0681
2 parents e4bb387 + 2526e5b
commit 2ad0681
Show file tree

Hide file tree

Showing 11 changed files with 385 additions and 257 deletions.
diff --git a/chart/Chart.yaml b/chart/Chart.yaml
@@ -2,7 +2,7 @@ apiVersion: v2
 name: metagrid
 description: A Helm chart for the Metagrid frontend/backend
 type: application
-version: 0.1.1
+version: 0.1.2
 appVersion: "v1.0.9-beta"
 home: https://github.com/esgf2-us/metagrid-k8s
 sources:

diff --git a/chart/README.md b/chart/README.md
@@ -52,6 +52,8 @@ helm delete my-release
 | external.keycloak.clientID | string | `"metagrid-llnl"` | Keycloak client ID |
 | external.hotjar | object | `{"id":null,"sv":null}` | [Hotjar](https://www.hotjar.com/) tracking codes, **OPTIONAL** |
 | external.googleAnalyticsTrackingID | string | `nil` | Google analytics tracking id, **OPTIONAL** |
+| projects | string | `nil` | Customize projects loaded during the initial migration, this is the value stored in [initial_projects_data.py](https://github.com/aims-group/metagrid/blob/master/backend/metagrid/initial_projects_data.py) |
+| imagePullSecrets | list | `[]` | List of secrets used to pull images from private registries |
 | django.replicaCount | int | `1` | Number of replicas |
 | django.debug | bool | `false` | Enable Django debugging |
 | django.adminURL | string | `"panel/"` | Relative path to the Django management panel |
@@ -61,49 +63,57 @@ helm delete my-release
 | django.image.repository | string | `"ghcr.io/aims-group/metagrid-backend"` | Django container URI |
 | django.image.pullPolicy | string | `"Always"` | Image pull policy |
 | django.image.tag | string | `"latest"` | Container tag |
-| django.imagePullSecrets | list | `[]` | List of image pull secrets |
 | django.nameOverride | string | `""` |  |
 | django.fullnameOverride | string | `""` |  |
-| django.migrateJob.enabled | bool | `true` | Enable database migration job |
 | django.podAnnotations | object | `{}` | Extra pod annotations |
-| django.podSecurityContext | object | `{}` | Pod security context |
-| django.securityContext | object | `{}` | Container security context |
-| django.resources | object | `{}` | Pod resources requests/limits |
+| django.migrateJob.enabled | bool | `true` | Enable database migration job |
+| django.migrateJob.restartPolicy | string | `"Never"` |  |
+| django.affinity | object | `{}` | [Affinity](https://kubernetes.io/docs/concepts/scheduling-eviction/assign-pod-node/#affinity-and-anti-affinity), pod node scheduling constraints |
+| django.resources | object | `{}` | Container [resources](https://kubernetes.io/docs/concepts/configuration/manage-resources-containers) |
+| django.securityContext | object | `{}` | Container [security](https://kubernetes.io/docs/tasks/configure-pod-container/security-context/#set-the-security-context-for-a-container) context |
+| django.nodeSelector | object | `{}` | Node [selector](https://kubernetes.io/docs/concepts/scheduling-eviction/assign-pod-node/#nodeselector) labels |
+| django.preemptionPolicy | string | `nil` | Pod [preemption](https://kubernetes.io/docs/concepts/scheduling-eviction/pod-priority-preemption/#preemption) policy |
+| django.priority | string | `nil` | Pod scheduling [priority](https://kubernetes.io/docs/concepts/scheduling-eviction/pod-priority-preemption/#pod-priority) |
+| django.priorityClassName | string | `nil` | Pod scheduling [priority](https://kubernetes.io/docs/concepts/scheduling-eviction/pod-priority-preemption/#priorityclass) class name |
+| django.podSecurityContext | object | `{}` | Pod [security](https://kubernetes.io/docs/tasks/configure-pod-container/security-context/#set-the-security-context-for-a-pod) context |
+| django.tolerations | list | `[]` | Pod [tolerations](https://kubernetes.io/docs/concepts/scheduling-eviction/taint-and-toleration) |
 | django.autoscaling | object | `{"enabled":false,"maxReplicas":100,"minReplicas":1,"targetCPUUtilizationPercentage":80}` | Horizontal pod autoscaling configuration |
-| django.nodeSelector | object | `{}` | Node selector |
-| django.tolerations | list | `[]` | Tolerations |
-| django.affinity | object | `{}` | Affinity |
 | django.service | object | `{"port":5000,"type":"ClusterIP"}` | Django service |
 | django.service.type | string | `"ClusterIP"` | Service type |
 | django.service.port | int | `5000` | Service port |
 | react.replicaCount | int | `1` | Number of replicas |
 | react.image.repository | string | `"ghcr.io/aims-group/metagrid-frontend"` | React container URI |
 | react.image.pullPolicy | string | `"Always"` | Container pull policy |
 | react.image.tag | string | `"latest"` | Container tag |
-| react.imagePullSecrets | list | `[]` | List of image pull secrets |
 | react.nameOverride | string | `""` |  |
 | react.fullnameOverride | string | `""` |  |
 | react.podAnnotations | object | `{}` | Extra pod annotations |
-| react.podSecurityContext | object | `{}` | Pod security context |
-| react.securityContext | object | `{}` | Container security context |
-| react.resources | object | `{}` | Resources requests/limits |
+| react.affinity | object | `{}` | [Affinity](https://kubernetes.io/docs/concepts/scheduling-eviction/assign-pod-node/#affinity-and-anti-affinity), pod node scheduling constraints |
+| react.resources | object | `{}` | Container [resources](https://kubernetes.io/docs/concepts/configuration/manage-resources-containers) |
+| react.securityContext | object | `{}` | Container [security](https://kubernetes.io/docs/tasks/configure-pod-container/security-context/#set-the-security-context-for-a-container) context |
+| react.nodeSelector | object | `{}` | Node [selector](https://kubernetes.io/docs/concepts/scheduling-eviction/assign-pod-node/#nodeselector) labels |
+| react.preemptionPolicy | string | `nil` | Pod [preemption](https://kubernetes.io/docs/concepts/scheduling-eviction/pod-priority-preemption/#preemption) policy |
+| react.priority | string | `nil` | Pod scheduling [priority](https://kubernetes.io/docs/concepts/scheduling-eviction/pod-priority-preemption/#pod-priority) |
+| react.priorityClassName | string | `nil` | Pod scheduling [priority](https://kubernetes.io/docs/concepts/scheduling-eviction/pod-priority-preemption/#priorityclass) class name |
+| react.podSecurityContext | object | `{}` | Pod [security](https://kubernetes.io/docs/tasks/configure-pod-container/security-context/#set-the-security-context-for-a-pod) context |
+| react.tolerations | list | `[]` | Pod [tolerations](https://kubernetes.io/docs/concepts/scheduling-eviction/taint-and-toleration) |
 | react.autoscaling | object | `{"enabled":false,"maxReplicas":100,"minReplicas":1,"targetCPUUtilizationPercentage":80}` | Horizontal Pod autoscaling configuration |
-| react.nodeSelector | object | `{}` | Node selector |
-| react.tolerations | list | `[]` | Tolerations |
-| react.affinity | object | `{}` | Affinity |
 | react.service | object | `{"port":3000,"type":"ClusterIP"}` | React service |
 | react.service.type | string | `"ClusterIP"` | Service type |
 | react.service.port | int | `3000` | Service port |
 | react.monitoring | object | `{"enabled":false}` | Prometheus monitoring |
 | ingress.enabled | bool | `false` | Enable ingress |
 | ingress.annotations | object | `{}` | Extra ingress annotations |
 | ingress.labels | object | `{}` | Extra ingress labels |
+| ingress.className | string | `nil` | Override ingress class |
 | ingress.react | object | `{"host":null,"path":"/metagrid"}` | React ingress endpoint |
 | ingress.react.host | string | `nil` | Endpoint host |
 | ingress.react.path | string | `"/metagrid"` | Endpoint path |
 | ingress.django | object | `{"host":null,"path":"/metagrid-backend"}` | Django ingress endpoint |
 | ingress.django.host | string | `nil` | Endpoint host |
 | ingress.django.path | string | `"/metagrid-backend"` | Endpoint path |
+| ingress.tls.enabled | bool | `false` |  |
+| ingress.tls.secretName | string | `nil` |  |
 | postgresql | object | `{"enabled":true,"persistence":{"enabled":false},"pgpool":{"adminPassword":"pgpooladminpass","containerSecurityContext":{"enabled":false},"podSecurityContext":{"enabled":false}},"postgresql":{"containerSecurityContext":{"enabled":false},"password":"pgpass","podSecurityContext":{"enabled":false},"replicaCount":1,"repmgrPassword":"repmgrpass"}}` | Postgresql database, **REQUIRED** |
 | postgresql.persistence | object | `{"enabled":false}` | Enabled persistence |
 | postgresql.postgresql.password | string | `"pgpass"` | Postgresql password, should replace the default with something more secure |
@@ -112,8 +122,41 @@ helm delete my-release
 | extraManifests | object | `{}` | Extra manifests |
 
 ## Configuration and installation details
-### External TLS terminatioj
-Some urls are auto-generated based on the configuration of the ingress. If TLS is not configured on the ingress (handled by upstream ingress) then this may result in some misconfiguration.
+### Setting the initial projects
+The `projects` value can be used to set the initial projects loaded by the backend. The following example will create a simple `CMIP6` project.
+
+```
+projects: |
+  group_description = {
+    "General": "Just a group",
+  }
+  projects = [
+    {
+      "name": "CMIP6",
+      ...
+      "facets_by_group": {
+        "General": ["data_node"],
+      },
+  ]
+```
+
+To add/remove projects after the initial migration, edit `projects` and upgrade the `Helm` chart. Next clear the existing database tables and run the migration update.
+
+#### Clear existing database tables
+```
+kubectl exec -it $(kubectl get pod -oname -l app.kubernetes.io/component=django) -- python manage.py migrate projects zero
+```
+
+#### Run the migration update
+```
+kubectl exec -it $(kubectl get pod -oname -l app.kubernetes.io/component=django) -- python manage.py migrate projects
+```
+
+### External TLS termination
+The `nodeStatus` and `metagridAPIUrl` values by default are automatically generated. If using an upstream reverse-proxy that handles TLS, then these urls will be incorrect and cause
+the application to not work correctly.
+
+To fix this both `nodeStatus` and `metagridAPIUrl` need to be overwritten, see the following example.
 
 ```
 external:

diff --git a/chart/README.md.gotmpl b/chart/README.md.gotmpl
@@ -29,8 +29,41 @@ helm delete my-release
 {{ template "chart.valuesSection" . }}
 
 ## Configuration and installation details
-### External TLS terminatioj
-Some urls are auto-generated based on the configuration of the ingress. If TLS is not configured on the ingress (handled by upstream ingress) then this may result in some misconfiguration.
+### Setting the initial projects
+The `projects` value can be used to set the initial projects loaded by the backend. The following example will create a simple `CMIP6` project.
+
+```
+projects: |
+  group_description = {
+    "General": "Just a group",
+  }
+  projects = [
+    {
+      "name": "CMIP6",
+      ...
+      "facets_by_group": {
+        "General": ["data_node"],
+      },
+  ]
+```
+
+To add/remove projects after the initial migration, edit `projects` and upgrade the `Helm` chart. Next clear the existing database tables and run the migration update.
+
+#### Clear existing database tables
+```
+kubectl exec -it $(kubectl get pod -oname -l app.kubernetes.io/component=django) -- python manage.py migrate projects zero
+```
+
+#### Run the migration update
+```
+kubectl exec -it $(kubectl get pod -oname -l app.kubernetes.io/component=django) -- python manage.py migrate projects
+```
+
+### External TLS termination
+The `nodeStatus` and `metagridAPIUrl` values by default are automatically generated. If using an upstream reverse-proxy that handles TLS, then these urls will be incorrect and cause
+the application to not work correctly.
+
+To fix this both `nodeStatus` and `metagridAPIUrl` need to be overwritten, see the following example.
 
 ```
 external:

diff --git a/chart/templates/_helpers.tpl b/chart/templates/_helpers.tpl
@@ -123,14 +123,132 @@ Django ESGF node status url
 */}}
 {{- define "metagrid.django.esgfNodeStatusUrl" -}}
 {{- $service := printf "127.0.0.1:%v" .Values.django.service.port }}
+{{- $ssl := ternary "s" "" .Values.ingress.tls.enabled }}
 {{- $host := ternary .Values.ingress.django.host $service .Values.ingress.enabled }}
-{{- $url := printf "http://%v/%v/proxy/status" $host (trimPrefix "/" .Values.ingress.django.path) }}
+{{- $url := printf "http%v://%v/%v/proxy/status" $ssl $host (trimPrefix "/" .Values.ingress.django.path) }}
 {{- printf "%v" (default $url .Values.external.nodeStatus) }}
 {{- end }}
 
 {{- define "metagrid.react.metagridUrl" -}}
 {{- $service := printf "127.0.0.1:%v" .Values.django.service.port }}
+{{- $ssl := ternary "s" "" .Values.ingress.tls.enabled }}
 {{- $host := ternary .Values.ingress.django.host $service .Values.ingress.enabled }}
-{{- $url := printf "http://%v/%v" $host .Values.ingress.django.path  }}
+{{- $url := printf "http%v://%v/%v" $ssl $host .Values.ingress.django.path  }}
 {{- printf "%v" (default $url .Values.external.metagridAPIUrl) }}
 {{- end }}
+
+{{- define "metagrid.podSpec" -}}
+{{- with .affinity }}
+affinity:
+{{- toYaml . | nindent 2 }}
+{{- end }}
+containers:
+- name: {{ .name }}
+  {{- with .args }}
+  args:
+  {{- toYaml . | nindent 4 }}
+  {{- end }}
+  {{- with .command }}
+  command:
+  {{- toYaml . | nindent 4 }}
+  {{- end }}
+  {{- with .env }}
+  env:
+  {{- range $name, $value := . }}
+  - name: {{ $name }}
+    value: {{ tpl $value .TemplateValues | quote }}
+  {{- end }}
+  {{- end }}
+  {{- with .envFrom }}
+  envFrom:
+  {{- toYaml . | nindent 4 }}
+  {{- end }}
+  image: {{ .image.repository }}:{{ .image.tag }}
+  {{- with .image.pullPolicy }}
+  imagePullPolicy: {{ . }}
+  {{- end }}
+  {{- with .livenessProbe }}
+  livenessProbe:
+  {{- toYaml . | nindent 4 }}
+  {{- end }}
+  {{- with .service }}
+  ports:
+  - name: {{ .name }}
+    containerPort: {{ .port }}
+    protocol: {{ default "TCP" .protocol }}
+  {{- end }}
+  {{- with .readinessProbe }}
+  readinessProbe:
+  {{- toYaml . | nindent 4 }}
+  {{- end }}
+  {{- with .resources }}
+  resources:
+  {{- toYaml . | nindent 4 }}
+  {{- end }}
+  {{- with .securityContext }}
+  securityContext:
+  {{- toYaml . | nindent 4 }}
+  {{- end }}
+  {{- with .startupProbe }}
+  startupProbe:
+  {{- toYaml . | nindent 4 }}
+  {{- end }}
+  {{- with .persistence }}
+  volumeMounts:
+  - mountPath: {{ .mountPath }}
+    name: {{ .name }}
+    readOnly: {{ default "false" .readOnly }}
+    {{- with .subPath }}
+    subPath: {{ . }}
+    {{- end }}
+  {{- end }}
+{{- with .dnsConfig }}
+dnsConfig:
+{{- toYaml . | nindent 2 }}
+{{- end }}
+{{- with .dnsPolicy }}
+dnsPolicy: {{ . }}
+{{- end }}
+{{- with .image.pullSecrets }}
+imagePullSecrets:
+- name: {{ include "metagrid.fullname" .TemplateValues }}
+{{- end }}
+{{- with .nodeSelector }}
+nodeSelector:
+{{- toYaml . | nindent 2 }}
+{{- end }}
+{{- with .preemptionPolicy }}
+preemptionPolicy: {{ . }}
+{{- end }}
+{{- with .priority }}
+priority: {{ . }}
+{{- end }}
+{{- with .priorityClassName }}
+priorityClassName: {{ . }}
+{{- end }}
+{{- with .restartPolicy }} 
+restartPolicy: {{ . }}
+{{- end }}
+{{- with .runtimeClassName }}
+runtimeClassName: {{ . }}
+{{- end }}
+{{- with .podSecurityContext }}
+securityContext:
+{{- toYaml . | nindent 2 }}
+{{- end }}
+{{- with .tolerations }}
+tolerations:
+{{- toYaml . | nindent 2 }}
+{{- end }}
+{{- with .persistence }}
+volumes:
+{{- if eq .type "configmap" }}
+- configMap:
+    name: {{ .resourceName }}
+{{- else if eq .type "secret" }}
+- secret:
+    secretName: {{ .resourceName }}
+{{- end }}
+  name: {{ .name }}
+{{- end }}
+{{- end }}
diff --git a/chart/templates/django/configmap.yaml b/chart/templates/django/configmap.yaml
@@ -12,7 +12,7 @@ data:
   PYTHONBUFFERED: "1"
   DJANGO_DEBUG: {{ .Values.django.debug | quote }}
   DJANGO_ADMIN_URL: {{ .Values.django.adminURL }}
-  DJANGO_ALLOWED_HOSTS: {{ include "metagrid.django_allowed_hosts" . }}
+  DJANGO_ALLOWED_HOSTS: {{ include "metagrid.django_allowed_hosts" . | quote }}
   DJANGO_SECURE_SSL_REDIRECT: "False"
   DJANGO_SETTINGS_MODULE: config.settings.production
   DOMAIN_NAME: {{ default "127.0.0.1" .Values.ingress.django.host }}

diff --git a/chart/templates/django/deployment.yaml b/chart/templates/django/deployment.yaml
@@ -23,50 +23,15 @@ spec:
         {{- include "metagrid.selectorLabels" . | nindent 8 }}
         app.kubernetes.io/component: django
     spec:
-      imagePullSecrets:
-      {{- range $name, $_ := .Values.imagePullSecrets }}
-      - name: {{ include "metagrid.fullname" $ }}-{{ regexReplaceAll "\\W+" $name "-" }}
-      {{- end }}
-      securityContext:
-        {{- toYaml .Values.django.podSecurityContext | nindent 8 }}
-      containers:
-        - name: {{ .Chart.Name }}
-          args: ["/start"]
-          securityContext:
-            {{- toYaml .Values.django.securityContext | nindent 12 }}
-          image: "{{ .Values.django.image.repository }}:{{ .Values.django.image.tag | default .Chart.AppVersion }}"
-          imagePullPolicy: {{ .Values.django.image.pullPolicy }}
-          envFrom:
-          - configMapRef:
-              name: {{ include "metagrid.fullname" . }}-django
-          - secretRef:
-              name: {{ include "metagrid.fullname" . }}-django
-          env:
-            {{- range $key, $value := default dict .Values.django.env}}
-            - name: {{ tpl $key $ }}
-              value: {{ tpl $value $ | quote }}
-            {{- end }}
-          ports:
-          - name: http
-            containerPort: {{ .Values.django.service.port }}
-            protocol: TCP
-          livenessProbe:
-            tcpSocket:
-              port: http
-          readinessProbe:
-            tcpSocket:
-              port: http
-          resources:
-            {{- toYaml .Values.django.resources | nindent 12 }}
-      {{- with .Values.django.nodeSelector }}
-      nodeSelector:
-        {{- toYaml . | nindent 8 }}
-      {{- end }}
-      {{- with .Values.django.affinity }}
-      affinity:
-        {{- toYaml . | nindent 8 }}
-      {{- end }}
-      {{- with .Values.django.tolerations }}
-      tolerations:
-        {{- toYaml . | nindent 8 }}
-      {{- end }}
+      {{- $django := deepCopy .Values.django }}
+      {{- $_ := set $django "name" "django" }}
+      {{- $_ := set $django.image "pullSecrets" .Values.imagePullSecrets }}
+      {{- $_ := set $django "args" (list "/start") }}
+      {{- $config := dict "configMapRef" (dict "name" (printf "%s-django" (include "metagrid.fullname" .))) }}
+      {{- $secret := dict "secretRef" (dict "name" (printf "%s-django" (include "metagrid.fullname" .))) }}
+      {{- $_ := set $django "envFrom" (list $config $secret) }}
+      {{- $_ := set $django "TemplateValues" . }}
+      {{- $_ := set $django.service "name" "http" }}
+      {{- $_ := set $django "livenessProbe" (dict "tcpSocket" (dict "port" "http")) }}
+      {{- $_ := set $django "readinessProbe" (dict "tcpSocket" (dict "port" "http")) }}
+      {{- include "metagrid.podSpec" $django | nindent 6 }}