feat(website): generate metadata docs from config (#3477)

* feat(website, config): Add code to generate metadata docs and description from config

docs/src/content/docs/reference/helm-chart-config.mdx

@@ -705,6 +705,24 @@ Each organism object has the following fields:
             <td></td>
             <td>Whether the field is required by backend</td>
         </tr>
+        <tr>
+            <td>`desired`</td>
+            <td>Boolean</td>
+            <td></td>
+            <td>Whether the field is a desired input field for submitters</td>
+        </tr>
+        <tr>
+            <td>`definition`</td>
+            <td>Boolean</td>
+            <td></td>
+            <td>Definition of input field for submitters</td>
+        </tr>
+        <tr>
+            <td>`guidance`</td>
+            <td>Boolean</td>
+            <td></td>
+            <td>Guidance for submitters on filling in input field</td>
+        </tr>
         <tr>
             <td>`noInput`</td>
             <td>Boolean</td>

kubernetes/loculus/templates/_common-metadata.tpl

@@ -180,6 +180,9 @@ organisms:
       inputFields: {{- include "loculus.inputFields" . | nindent 8 }}
         - name: versionComment
           displayName: Version comment
+          definition: "Reason for revising sequences or other general comments concerning a specific version"
+          example: "Fixed an issue in previous version where low-coverage nucleotides were erroneously filled with reference sequence"
+          desired: true
       metadata:
         {{- $args := dict "metadata" (concat $commonMetadata .metadata) "nucleotideSequences" $nucleotideSequences}}
         {{ $metadata := include "loculus.generateWebsiteMetadata" $args | fromYaml }}

kubernetes/loculus/templates/_inputFieldsFromValues.tpl

@@ -2,7 +2,7 @@
 {{- $data := . }}
 {{- $metadata := $data.metadata }}
 {{- $extraFields := $data.extraInputFields }}
-{{- $TO_KEEP := list "name" "displayName" "definition" "guidance" "example" "required" "noEdit"}}
+{{- $TO_KEEP := list "name" "displayName" "definition" "guidance" "example" "required" "noEdit" "desired"}}
 
 
 {{- $fieldsDict := dict }}

website/src/components/MetadataTable.astro

@@ -0,0 +1,17 @@
+---
+import { getConfiguredOrganisms, getSchema } from '../config';
+import OrganismTableSelector from './OrganismMetadataTableSelector';
+import type { OrganismMetadata } from './OrganismMetadataTableSelector';
+
+const configuredOrganisms = getConfiguredOrganisms();
+const organisms: OrganismMetadata[] = configuredOrganisms.map((organism) => {
+    return {
+        key: organism.key,
+        displayName: organism.displayName,
+        metadata: getSchema(organism.key).metadata,
+        inputFields: getSchema(organism.key).inputFields,
+    };
+});
+---
+
+<OrganismTableSelector organisms={organisms} client:load />

website/src/components/OrganismMetadataTableSelector.tsx

@@ -0,0 +1,131 @@
+import React, { useState, useEffect } from 'react';
+import type { FC } from 'react';
+
+import { routes } from '../routes/routes.ts';
+import type { Metadata, InputField } from '../types/config.ts';
+import { groupFieldsByHeader } from '../utils/groupFieldsByHeader.ts';
+import IwwaArrowDown from '~icons/iwwa/arrow-down';
+
+export type OrganismMetadata = {
+    key: string;
+    displayName: string;
+    metadata: Metadata[];
+    inputFields: InputField[];
+};
+
+type Props = {
+    organisms: OrganismMetadata[];
+};
+
+const OrganismMetadataTableSelector: FC<Props> = ({ organisms }) => {
+    const [selectedOrganism, setSelectedOrganism] = useState<OrganismMetadata | null>(null);
+    const [groupedFields, setGroupedFields] = useState<Map<string, InputField[]>>(new Map());
+    const [expandedHeaders, setExpandedHeaders] = useState<Set<string>>(new Set(['Required fields', 'Desired fields']));
+
+    const handleOrganismSelect = (event: { target: { value: string } }) => {
+        const organismKey = event.target.value;
+        const organism = organisms.find((o) => o.key === organismKey);
+        setSelectedOrganism(organism ?? null);
+    };
+
+    const toggleHeader = (header: string) => {
+        const updatedExpandedHeaders = new Set(expandedHeaders);
+        if (updatedExpandedHeaders.has(header)) {
+            updatedExpandedHeaders.delete(header); // Close the table if already expanded
+        } else {
+            updatedExpandedHeaders.add(header);
+        }
+        setExpandedHeaders(updatedExpandedHeaders);
+    };
+
+    useEffect(() => {
+        if (selectedOrganism) {
+            setGroupedFields(groupFieldsByHeader(selectedOrganism.inputFields, selectedOrganism.metadata));
+        }
+    }, [selectedOrganism]);
+
+    return (
+        <div>
+            <div>
+                <select id='organism-select' onChange={handleOrganismSelect} className='border border-gray-300 p-2'>
+                    <option value=''>-- Select an Organism --</option>
+                    {organisms.map((organism) => (
+                        <option key={organism.key} value={organism.key}>
+                            {organism.displayName}
+                        </option>
+                    ))}
+                </select>
+            </div>
+
+            {selectedOrganism && (
+                <div className='mt-6'>
+                    <h1 className='text-2xl font-bold mb-4'>{selectedOrganism.displayName}</h1>
+                    <div>
+                        You can download all metadata fields and their descriptions here:{' '}
+                        <a
+                            href={routes.metadataOverview(selectedOrganism.key)}
+                            className='text-primary-700  opacity-90'
+                        >
+                            metadata_fields_descriptions.csv
+                        </a>
+                    </div>
+                    {Array.from(groupedFields.entries()).map(([header, fields]) => (
+                        <div key={header} className='mb-8'>
+                            <h3
+                                className='text-lg font-semibold mb-4 cursor-pointer'
+                                onClick={() => toggleHeader(header)}
+                            >
+                                {header}
+                                <IwwaArrowDown className='inline-block -mt-1 ml-1 h-4 w-4' />
+                            </h3>
+                            <div
+                                className={`transition-all duration-300 ${
+                                    expandedHeaders.has(header) ? 'block' : 'sr-only'
+                                }`}
+                                data-table-header={header}
+                            >
+                                <MetadataTable fields={fields} metadata={selectedOrganism.metadata} />
+                            </div>
+                        </div>
+                    ))}
+                </div>
+            )}
+        </div>
+    );
+};
+
+export default OrganismMetadataTableSelector;
+
+type TableProps = {
+    fields: InputField[];
+    metadata: Metadata[];
+};
+
+const MetadataTable: FC<TableProps> = ({ fields, metadata }) => {
+    return (
+        <table className='table-auto border-collapse border border-gray-200 w-full'>
+            <thead>
+                <tr>
+                    <th className='border border-gray-300 px-4 py-2 w-[20%]'>Field Name</th>
+                    <th className='border border-gray-300 px-4 py-2 w-[13%]'>Type</th>
+                    <th className='border border-gray-300 px-4 py-2 w-[37%]'>Description</th>
+                    <th className='border border-gray-300 px-4 py-2 w-[30%]'>Example</th>
+                </tr>
+            </thead>
+            <tbody>
+                {fields.map((field) => {
+                    const metadataEntry = metadata.find((meta) => meta.name === field.name);
+
+                    return (
+                        <tr key={field.name}>
+                            <td className='border border-gray-300 px-4 py-2'>{field.name}</td>
+                            <td className='border border-gray-300 px-4 py-2'>{metadataEntry?.type ?? 'String'}</td>
+                            <td className='border border-gray-300 px-4 py-2'>{`${field.definition ?? ''} ${field.guidance ?? ''}`}</td>
+                            <td className='border border-gray-300 px-4 py-2'>{field.example ?? ''}</td>
+                        </tr>
+                    );
+                })}
+            </tbody>
+        </table>
+    );
+};

website/src/pages/[organism]/metadata-overview/index.ts

@@ -0,0 +1,38 @@
+import type { APIRoute } from 'astro';
+
+import { cleanOrganism } from '../../../components/Navigation/cleanOrganism.ts';
+import { getSchema } from '../../../config.ts';
+import { SUBMISSION_ID_FIELD } from '../../../settings.ts';
+
+export const GET: APIRoute = ({ params }) => {
+    const rawOrganism = params.organism!;
+    const { organism } = cleanOrganism(rawOrganism);
+    if (organism === undefined) {
+        return new Response(undefined, {
+            status: 404,
+        });
+    }
+
+    const extraFields = [SUBMISSION_ID_FIELD];
+
+    const tableHeader = 'Field Name\tRequired\tDefinition\tGuidance\tExample';
+
+    const { inputFields } = getSchema(organism.key);
+
+    const headers: Record<string, string> = {
+        'Content-Type': 'text/tsv', // eslint-disable-line @typescript-eslint/naming-convention
+    };
+
+    const filename = `${organism.displayName.replaceAll(' ', '_')}_metadata_overview.tsv`;
+    headers['Content-Disposition'] = `attachment; filename="${filename}"`;
+
+    const fieldNames = inputFields.map(
+        (field) =>
+            `${field.name}\t${field.required ?? ''}\t${field.definition ?? ''} ${field.guidance ?? ''}\t${field.example ?? ''}`,
+    );
+    const tsvTemplate = [tableHeader, ...extraFields, ...fieldNames].join('\n');
+
+    return new Response(tsvTemplate, {
+        headers,
+    });
+};

website/src/routes/routes.ts

@@ -14,6 +14,7 @@ export const routes = {
     searchPage: (organism: string) => withOrganism(organism, `/search`),
     metadataTemplate: (organism: string, format: UploadAction) =>
         withOrganism(organism, `/submission/template?format=${format}`),
+    metadataOverview: (organism: string) => withOrganism(organism, `/metadata-overview`),
 
     mySequencesPage: (organism: string, groupId: number) =>
         SubmissionRouteUtils.toUrl({

website/src/types/config.ts

@@ -63,6 +63,11 @@ export const inputField = z.object({
     name: z.string(),
     displayName: z.string().optional(),
     noEdit: z.boolean().optional(),
+    required: z.boolean().optional(),
+    definition: z.string().optional(), // Definition, Example and Guidance for submitters
+    example: z.union([z.string(), z.number()]).optional(),
+    guidance: z.string().optional(),
+    desired: z.boolean().optional(),
 });
 
 export type InputField = z.infer<typeof inputField>;

website/src/utils/groupFieldsByHeader.ts

@@ -0,0 +1,37 @@
+import type { InputField, Metadata } from '../types/config';
+
+const SUBMISSION_ID_FIELD: InputField = {
+    name: 'submissionId',
+    displayName: 'Submission ID',
+    definition: 'FASTA ID',
+    guidance:
+        'Your sequence identifier; should match the FASTA file header - this is used to link the metadata to the FASTA sequence',
+    example: 'GJP123',
+    noEdit: true,
+    required: true,
+};
+
+export const groupFieldsByHeader = (inputFields: InputField[], metadata: Metadata[]): Map<string, InputField[]> => {
+    const groups = new Map<string, InputField[]>();
+
+    const requiredFields = inputFields.filter((meta) => meta.required);
+    const desiredFields = inputFields.filter((meta) => meta.desired);
+
+    groups.set('Required fields', [...requiredFields, SUBMISSION_ID_FIELD]);
+    groups.set('Desired fields', desiredFields);
+    groups.set('Submission details', [SUBMISSION_ID_FIELD]);
+
+    inputFields.forEach((field) => {
+        const metadataEntry = metadata.find((meta) => meta.name === field.name);
+        const header = metadataEntry?.header ?? 'Uncategorized';
+
+        if (!groups.has(header)) {
+            groups.set(header, []);
+        }
+        groups.get(header)!.push({
+            ...field,
+        });
+    });
+
+    return groups;
+};