Add segmented as a config param

loculus-project · May 22, 2024 · 4756ffe · 4756ffe
1 parent c94ba9f
commit 4756ffe
Show file tree

Hide file tree

Showing 6 changed files with 215 additions and 106 deletions.
diff --git a/ingest/config/config.yaml b/ingest/config/config.yaml
@@ -2,3 +2,12 @@ taxon_id: 186538
 backend_url: https://backend-main.loculus.org/
 keycloak_token_url: https://authentication-main.loculus.org/realms/loculus/protocol/openid-connect/token
 organism: ebola-zaire
+
+# taxon_id:  3052518
+# backend_url: http://localhost:8079/
+# keycloak_token_url: http://localhost:8083/realms/loculus/protocol/openid-connect/token
+# organism: cchf
+# nucleotideSequences:
+#   - M
+#   - L
+#   - S
diff --git a/kubernetes/loculus/templates/_common-metadata.tpl b/kubernetes/loculus/templates/_common-metadata.tpl
@@ -99,6 +99,7 @@ organisms:
       {{- with ($instance.schema | include "loculus.patchMetadataSchema" | fromYaml) }}
       instanceName: {{ quote .instanceName }}
       loadSequencesAutomatically: {{ .loadSequencesAutomatically | default false }}
+      {{- $segmented := (.segmented | default false )}}
       {{ if .image }}
       image: {{ .image }}
       {{ end }}
@@ -108,10 +109,8 @@ organisms:
       primaryKey: accessionVersion
       inputFields: {{- include "loculus.inputFields" . | nindent 8 }}
       metadata:
-        {{ $metadata := concat $commonMetadata .metadata
-            | include "loculus.generateWebsiteMetadata"
-            | fromYaml
-         }}
+        {{- $args := dict "metadata" .metadata "segmented" $segmented }}
+        {{ $metadata := include "loculus.generateWebsiteMetadata" $args | fromYaml }}
         {{ $metadata.fields | toYaml | nindent 8 }}
       {{ .website | toYaml | nindent 6 }}
       {{- end }}
@@ -123,39 +122,81 @@ organisms:
 {{/* Generate website metadata from passed metadata array */}}
 {{- define "loculus.generateWebsiteMetadata" }}
 fields:
-{{- range . }}
-  - name: {{ quote .name }}
-    type: {{ .type | default "string" | quote }}
-    {{- if .autocomplete }}
-    autocomplete: {{ .autocomplete }}
-    {{- end }}
-    {{- if .notSearchable }}
-    notSearchable: {{ .notSearchable }}
-    {{- end }}
-    {{- if .initiallyVisible }}
-    initiallyVisible: {{ .initiallyVisible }}
-    {{- end }}
-    {{- if or (or (eq .type "timestamp")  (eq .type "date")) ( .rangeSearch) }}
-    rangeSearch: true
-    {{- end }}
-    {{- if .hideOnSequenceDetailsPage }}
-    hideOnSequenceDetailsPage: {{ .hideOnSequenceDetailsPage }}
-    {{- end }}
-    {{- if .displayName }}
-    displayName: {{ quote .displayName }}
-    {{- end }}
-    {{- if .truncateColumnDisplayTo }}
-    truncateColumnDisplayTo: {{ .truncateColumnDisplayTo }}
-    {{- end }}
-    {{- if .customDisplay }}
-    customDisplay:
-      type: {{ quote .customDisplay.type }}
-      url: {{ .customDisplay.url }}
-    {{- end }}
-    header: {{ default "Other" .header }}
+{{- $metadataList := .metadata }}
+{{- $use_segments := .segmented }}
+{{- $segments := list "M" "S" "L" }}
+{{- range $metadataList }}
+{{- $currentItem := . }}
+{{- if and $use_segments .segmented }}
+  {{- range $segment := $segments }}
+    - name: {{ printf "%s_%s" $currentItem.name $segment | quote }}
+      type: {{ $currentItem.type | default "string" | quote }}
+      {{- if $currentItem.autocomplete }}
+      autocomplete: {{ $currentItem.autocomplete }}
+      {{- end }}
+      {{- if $currentItem.notSearchable }}
+      notSearchable: {{ $currentItem.notSearchable }}
+      {{- end }}
+      {{- if $currentItem.initiallyVisible }}
+      initiallyVisible: {{ $currentItem.initiallyVisible }}
+      {{- end }}
+      {{- if or (or (eq $currentItem.type "timestamp")  (eq $currentItem.type "date")) ( $currentItem.rangeSearch) }}
+      rangeSearch: true
+      {{- end }}
+      {{- if $currentItem.hideOnSequenceDetailsPage }}
+      hideOnSequenceDetailsPage: {{ $currentItem.hideOnSequenceDetailsPage }}
+      {{- end }}
+      {{- if $currentItem.displayName }}
+      displayName: {{ quote $currentItem.displayName }}
+      {{- end }}
+      {{- if $currentItem.truncateColumnDisplayTo }}
+      truncateColumnDisplayTo: {{ $currentItem.truncateColumnDisplayTo }}
+      {{- end }}
+      {{- if $currentItem.customDisplay }}
+      customDisplay:
+        type: {{ quote $currentItem.customDisplay.type }}
+        url: {{ $currentItem.customDisplay.url }}
+      {{- end }}
+      header: {{ default "Other" $currentItem.header }}
+  {{- end}}
+{{- else }}
+    - name: {{ quote .name }}
+      type: {{ .type | default "string" | quote }}
+      {{- if .autocomplete }}
+      autocomplete: {{ .autocomplete }}
+      {{- end }}
+      {{- if .notSearchable }}
+      notSearchable: {{ .notSearchable }}
+      {{- end }}
+      {{- if .initiallyVisible }}
+      initiallyVisible: {{ .initiallyVisible }}
+      {{- end }}
+      {{- if .segmented }}
+      segmented: {{ .segmented }}
+      {{- end }}
+      {{- if or (or (eq .type "timestamp")  (eq .type "date")) ( .rangeSearch) }}
+      rangeSearch: true
+      {{- end }}
+      {{- if .hideOnSequenceDetailsPage }}
+      hideOnSequenceDetailsPage: {{ .hideOnSequenceDetailsPage }}
+      {{- end }}
+      {{- if .displayName }}
+      displayName: {{ quote .displayName }}
+      {{- end }}
+      {{- if .truncateColumnDisplayTo }}
+      truncateColumnDisplayTo: {{ .truncateColumnDisplayTo }}
+      {{- end }}
+      {{- if .customDisplay }}
+      customDisplay:
+        type: {{ quote .customDisplay.type }}
+        url: {{ .customDisplay.url }}
+      {{- end }}
+      header: {{ default "Other" .header }}
+{{- end}}
 {{- end}}
 {{- end}}
 
+
 {{/* Generate backend config from passed config object */}}
 {{- define "loculus.generateBackendConfig" }}
 accessionPrefix: {{ quote $.Values.accessionPrefix }}
@@ -167,12 +208,11 @@ organisms:
   {{ $key }}:
     schema:
       {{- with $instance.schema }}
+      {{- $segmented := (.segmented | default false )}}
       instanceName: {{ quote .instanceName }}
       metadata:
-        {{ $metadata := (include "loculus.patchMetadataSchema" .
-          | fromYaml).metadata
-          | include "loculus.generateBackendMetadata"
-          | fromYaml }}
+        {{- $args := dict "metadata" (include "loculus.patchMetadataSchema" . | fromYaml).metadata "segmented" $segmented }}
+        {{ $metadata := include "loculus.generateBackendMetadata" $args | fromYaml }}
         {{ $metadata.fields | toYaml | nindent 8 }}
       {{- end }}
     referenceGenomes:
@@ -183,14 +223,29 @@ organisms:
 {{/* Generate backend metadata from passed metadata array */}}
 {{- define "loculus.generateBackendMetadata" }}
 fields:
-{{- range . }}
+{{- $metadataList := .metadata }}
+{{- $use_segments := .segmented }}
+{{- $segments := list "M" "S" "L" }}
+{{- range $metadataList }}
+{{- $currentItem := . }}
+{{- $segmented := (.segmented | default false )}}
+{{- if and $use_segments $segmented }}
+{{- range $segment := $segments }}
+  - name: {{ printf "%s_%s" $currentItem.name $segment | quote }}
+    type: {{ $currentItem.type | default "string" | quote }}
+    {{- if $currentItem.required }}
+    required: {{ $currentItem.required }}
+    {{- end }}
+{{- end}}
+{{- else }}
   - name: {{ quote .name }}
     type: {{ .type | default "string" | quote }}
     {{- if .required }}
     required: {{ .required }}
     {{- end }}
 {{- end}}
 {{- end}}
+{{- end}}
 
 {{- define "loculus.publicRuntimeConfig" }}
             {{- if $.Values.codespaceName }}

diff --git a/kubernetes/loculus/templates/_preprocessingFromValues.tpl b/kubernetes/loculus/templates/_preprocessingFromValues.tpl
@@ -1,5 +1,6 @@
 {{- define "loculus.preprocessingSpecs" -}}
-{{- $metadata := . }}
+{{- $metadata := .metadata }}
+{{- $use_segments := .segmented }}
 {{- $specs := dict }}
 
 {{- range $field := $metadata }}
@@ -15,6 +16,13 @@
   {{- end }}
 {{- end }}
 
+{{- if $use_segments }}
+{{- if hasKey $field "segmented" }}
+  {{- $segmented := index $field "segmented" }}
+  {{- $_ := set $spec "args" (dict "segmented" true) }}
+{{- end }}
+{{- end }}
+
 {{- if hasKey $field "preprocessing" }}
   {{- $preprocessing := index $field "preprocessing" }}
   {{- if eq (typeOf $preprocessing) "string" }}

diff --git a/kubernetes/loculus/templates/loculus-preprocessing-config.yaml b/kubernetes/loculus/templates/loculus-preprocessing-config.yaml
@@ -1,5 +1,6 @@
 {{- range $organism, $organismConfig := (.Values.organisms | default .Values.defaultOrganisms) }}
 {{- $metadata := ($organismConfig.schema | include "loculus.patchMetadataSchema" | fromYaml).metadata }}
+{{- $segmented := (($organismConfig.schema | include "loculus.patchMetadataSchema" | fromYaml).segmented | default false ) }}
 {{- range $processingIndex, $processingConfig := $organismConfig.preprocessing }}
 {{- if $processingConfig.configFile }}
 ---
@@ -11,7 +12,8 @@ data:
   preprocessing-config.yaml: |
     {{- $processingConfig.configFile | toYaml | nindent 4 }}
     processing_spec: 
-      {{- include "loculus.preprocessingSpecs" $metadata | nindent 6 }}
+      {{- $args := dict "metadata" $metadata "segmented" $segmented }}
+      {{- include "loculus.preprocessingSpecs" $args | nindent 6 }}
     organism: {{ $organism }}
 {{- end }}
 {{- end }}

diff --git a/kubernetes/loculus/values.yaml b/kubernetes/loculus/values.yaml
@@ -143,11 +143,13 @@ defaultOrganismConfig: &defaultOrganismConfig
         header: "INSDC"
         hideOnSequenceDetailsPage: true
         noInput: true
+        segmented: true
       - name: insdc_version
         type: int
         header: "INSDC"
         hideOnSequenceDetailsPage: true
         noInput: true
+        segmented: true
       - name: insdc_accession_full
         displayName: INSDC accession
         customDisplay:
@@ -156,18 +158,21 @@ defaultOrganismConfig: &defaultOrganismConfig
         header: "INSDC"
         ingest: genbank_accession
         noInput: true
+        segmented: true
       - name: bioproject_accessions
         customDisplay:
           type: link
           url: "https://www.ncbi.nlm.nih.gov/bioproject/__value__"
         header: "INSDC"
         ingest: bioprojects
+        segmented: true
       - name: biosample_accession
         customDisplay:
           type: link
           url: "https://www.ncbi.nlm.nih.gov/biosample/__value__"
         header: "INSDC" 
         noInput: true
+        segmented: true
       - name: culture_id
         displayName: Culture ID
         header: Sample details
@@ -692,11 +697,13 @@ defaultOrganismConfig: &defaultOrganismConfig
         autocomplete: true
         header: "INSDC"
         noInput: true
+        segmented: true
       - name: length
         type: int
         header: "Alignment states and QC metrics"
         noInput: true
         rangeSearch: true
+        segmented: true
       - name: host_name_scientific
         generateIndex: true
         autocomplete: true
@@ -1190,6 +1197,7 @@ defaultOrganisms:
     schema:
       <<: *schema
       instanceName: "Crimean-Congo Hemorrhagic Fever Virus"
+      segmented: true
       image: "https://upload.wikimedia.org/wikipedia/commons/thumb/5/5e/Crimean-Congo_Hemorrhagic_Fever_%28CCHF%29_Virus_%2840689899455%29.jpg/1920px-Crimean-Congo_Hemorrhagic_Fever_%28CCHF%29_Virus_%2840689899455%29.jpg"
       website:
         <<: *website