From ee6ec026d70ee53bab62209a8796224609108185 Mon Sep 17 00:00:00 2001 From: Felix Hennig Date: Thu, 12 Dec 2024 13:40:11 +0100 Subject: [PATCH] feat(backend): Return `pipelineVersion` in `get-released-data` (#3433) * Add 'pipeline_version' to SequenceEntriesView * Add 'pipelineVersion' to RawProcessedData * Add 'pipelineVersion' to get-released-data return values * Add 'pipelineVersion' to _common-metadata.tpl * Test fix * Add missing comma * Update schema documentation based on migration changes * trigger preview * fixes * Update schema documentation based on migration changes * trigger preview --------- Co-authored-by: GitHub Action --- backend/docs/db/schema.sql | 5 +++ .../backend/model/ReleasedDataModel.kt | 1 + .../service/submission/SequenceEntriesView.kt | 1 + .../submission/SubmissionDatabaseService.kt | 3 ++ ...ence_entries_view_add_pipeline_version.sql | 41 +++++++++++++++++++ .../submission/GetReleasedDataEndpointTest.kt | 6 ++- .../utils/EarliestReleaseDateFinderTest.kt | 1 + .../loculus/templates/_common-metadata.tpl | 4 ++ 8 files changed, 61 insertions(+), 1 deletion(-) create mode 100644 backend/src/main/resources/db/migration/V1.9__update_sequence_entries_view_add_pipeline_version.sql diff --git a/backend/docs/db/schema.sql b/backend/docs/db/schema.sql index 4230c9023..5898aa164 100644 --- a/backend/docs/db/schema.sql +++ b/backend/docs/db/schema.sql @@ -464,6 +464,11 @@ CREATE VIEW public.sequence_entries_view AS sepd.finished_processing_at, sepd.processed_data, (sepd.processed_data || em.joint_metadata) AS joint_metadata, + CASE + WHEN se.is_revocation THEN ( SELECT current_processing_pipeline.version + FROM public.current_processing_pipeline) + ELSE sepd.pipeline_version + END AS pipeline_version, sepd.errors, sepd.warnings, CASE diff --git a/backend/src/main/kotlin/org/loculus/backend/model/ReleasedDataModel.kt b/backend/src/main/kotlin/org/loculus/backend/model/ReleasedDataModel.kt index 591a63f38..3cccc63b8 100644 --- a/backend/src/main/kotlin/org/loculus/backend/model/ReleasedDataModel.kt +++ b/backend/src/main/kotlin/org/loculus/backend/model/ReleasedDataModel.kt @@ -128,6 +128,7 @@ open class ReleasedDataModel( ("versionStatus" to TextNode(versionStatus.name)), ("dataUseTerms" to TextNode(currentDataUseTerms.type.name)), ("dataUseTermsRestrictedUntil" to restrictedDataUseTermsUntil), + ("pipelineVersion" to LongNode(rawProcessedData.pipelineVersion)), ) + if (rawProcessedData.isRevocation) { mapOf("versionComment" to TextNode(rawProcessedData.versionComment)) diff --git a/backend/src/main/kotlin/org/loculus/backend/service/submission/SequenceEntriesView.kt b/backend/src/main/kotlin/org/loculus/backend/service/submission/SequenceEntriesView.kt index d62badefc..3d53b8848 100644 --- a/backend/src/main/kotlin/org/loculus/backend/service/submission/SequenceEntriesView.kt +++ b/backend/src/main/kotlin/org/loculus/backend/service/submission/SequenceEntriesView.kt @@ -46,6 +46,7 @@ object SequenceEntriesView : Table(SEQUENCE_ENTRIES_VIEW_NAME) { val versionCommentColumn = varchar("version_comment", 255).nullable() val errorsColumn = jacksonSerializableJsonb>("errors").nullable() val warningsColumn = jacksonSerializableJsonb>("warnings").nullable() + val pipelineVersionColumn = long("pipeline_version").nullable() override val primaryKey = PrimaryKey(accessionColumn, versionColumn) diff --git a/backend/src/main/kotlin/org/loculus/backend/service/submission/SubmissionDatabaseService.kt b/backend/src/main/kotlin/org/loculus/backend/service/submission/SubmissionDatabaseService.kt index 330a9de33..07d68a15c 100644 --- a/backend/src/main/kotlin/org/loculus/backend/service/submission/SubmissionDatabaseService.kt +++ b/backend/src/main/kotlin/org/loculus/backend/service/submission/SubmissionDatabaseService.kt @@ -583,6 +583,7 @@ class SubmissionDatabaseService( SequenceEntriesView.submittedAtTimestampColumn, SequenceEntriesView.releasedAtTimestampColumn, SequenceEntriesView.submissionIdColumn, + SequenceEntriesView.pipelineVersionColumn, DataUseTermsTable.dataUseTermsTypeColumn, DataUseTermsTable.restrictedUntilColumn, ) @@ -610,6 +611,7 @@ class SubmissionDatabaseService( null -> emptyProcessedDataProvider.provide(organism) else -> processedDataPostprocessor.retrieveFromStoredValue(processedData, organism) }, + pipelineVersion = it[SequenceEntriesView.pipelineVersionColumn]!!, submittedAtTimestamp = it[SequenceEntriesView.submittedAtTimestampColumn], releasedAtTimestamp = it[SequenceEntriesView.releasedAtTimestampColumn]!!, dataUseTerms = DataUseTerms.fromParameters( @@ -1206,5 +1208,6 @@ data class RawProcessedData( val releasedAtTimestamp: LocalDateTime, val submissionId: String, val processedData: ProcessedData, + val pipelineVersion: Long, val dataUseTerms: DataUseTerms, ) : AccessionVersionInterface diff --git a/backend/src/main/resources/db/migration/V1.9__update_sequence_entries_view_add_pipeline_version.sql b/backend/src/main/resources/db/migration/V1.9__update_sequence_entries_view_add_pipeline_version.sql new file mode 100644 index 000000000..bc233cadf --- /dev/null +++ b/backend/src/main/resources/db/migration/V1.9__update_sequence_entries_view_add_pipeline_version.sql @@ -0,0 +1,41 @@ +drop view if exists sequence_entries_view; + +create view sequence_entries_view as +select + se.*, + sepd.started_processing_at, + sepd.finished_processing_at, + sepd.processed_data as processed_data, + sepd.processed_data || em.joint_metadata as joint_metadata, + case + when se.is_revocation then (select version from current_processing_pipeline) + else sepd.pipeline_version + end as pipeline_version, + sepd.errors, + sepd.warnings, + case + when se.released_at is not null then 'APPROVED_FOR_RELEASE' + when se.is_revocation then 'PROCESSED' + when sepd.processing_status = 'IN_PROCESSING' then 'IN_PROCESSING' + when sepd.processing_status = 'PROCESSED' then 'PROCESSED' + else 'RECEIVED' + end as status, + case + when sepd.processing_status = 'IN_PROCESSING' then null + when sepd.errors is not null and jsonb_array_length(sepd.errors) > 0 then 'HAS_ERRORS' + when sepd.warnings is not null and jsonb_array_length(sepd.warnings) > 0 then 'HAS_WARNINGS' + else 'NO_ISSUES' + end as processing_result +from + sequence_entries se + left join sequence_entries_preprocessed_data sepd on + se.accession = sepd.accession + and se.version = sepd.version + and sepd.pipeline_version = (select version from current_processing_pipeline) + left join external_metadata_view em on + se.accession = em.accession + and se.version = em.version; + +update sequence_entries_preprocessed_data +set processing_status = 'PROCESSED' +where processing_status in ('HAS_ERRORS', 'FINISHED'); diff --git a/backend/src/test/kotlin/org/loculus/backend/controller/submission/GetReleasedDataEndpointTest.kt b/backend/src/test/kotlin/org/loculus/backend/controller/submission/GetReleasedDataEndpointTest.kt index 400954538..a6b7cabc2 100644 --- a/backend/src/test/kotlin/org/loculus/backend/controller/submission/GetReleasedDataEndpointTest.kt +++ b/backend/src/test/kotlin/org/loculus/backend/controller/submission/GetReleasedDataEndpointTest.kt @@ -50,6 +50,7 @@ import org.loculus.backend.controller.DEFAULT_GROUP_CHANGED import org.loculus.backend.controller.DEFAULT_GROUP_NAME import org.loculus.backend.controller.DEFAULT_GROUP_NAME_CHANGED import org.loculus.backend.controller.DEFAULT_ORGANISM +import org.loculus.backend.controller.DEFAULT_PIPELINE_VERSION import org.loculus.backend.controller.DEFAULT_USER_NAME import org.loculus.backend.controller.EndpointTest import org.loculus.backend.controller.datauseterms.DataUseTermsControllerClient @@ -143,7 +144,7 @@ class GetReleasedDataEndpointTest( responseBody.forEach { val id = it.metadata["accession"]!!.asText() val version = it.metadata["version"]!!.asLong() - assertThat(version, `is`(1L)) + assertThat(version, `is`(1)) val expectedMetadata = defaultProcessedData.metadata + mapOf( "accession" to TextNode(id), @@ -157,6 +158,7 @@ class GetReleasedDataEndpointTest( "releasedDate" to TextNode(currentDate), "submittedDate" to TextNode(currentDate), "dataUseTermsRestrictedUntil" to NullNode.getInstance(), + "pipelineVersion" to IntNode(DEFAULT_PIPELINE_VERSION.toInt()), ) for ((key, value) in it.metadata) { @@ -294,6 +296,7 @@ class GetReleasedDataEndpointTest( value, `is`(TextNode("This is a test revocation")), ) + "pipelineVersion" -> assertThat(value, `is`(IntNode(DEFAULT_PIPELINE_VERSION.toInt()))) else -> assertThat("value for $key", value, `is`(NullNode.instance)) } @@ -367,6 +370,7 @@ class GetReleasedDataEndpointTest( .andGetGroupId() SequenceEntriesTable.batchInsert(accessionVersions.shuffled()) { (accession, version) -> + this[SequenceEntriesTable.isRevocationColumn] = true this[SequenceEntriesTable.accessionColumn] = accession this[SequenceEntriesTable.versionColumn] = version this[SequenceEntriesTable.groupIdColumn] = submittingGroupId diff --git a/backend/src/test/kotlin/org/loculus/backend/utils/EarliestReleaseDateFinderTest.kt b/backend/src/test/kotlin/org/loculus/backend/utils/EarliestReleaseDateFinderTest.kt index 79d020f2b..bf0cf7e9c 100644 --- a/backend/src/test/kotlin/org/loculus/backend/utils/EarliestReleaseDateFinderTest.kt +++ b/backend/src/test/kotlin/org/loculus/backend/utils/EarliestReleaseDateFinderTest.kt @@ -71,6 +71,7 @@ fun row( groupId = 0, groupName = "foo", dataUseTerms = DataUseTerms.Open, + pipelineVersion = 0, ) // Notes: diff --git a/kubernetes/loculus/templates/_common-metadata.tpl b/kubernetes/loculus/templates/_common-metadata.tpl index 6889c8e28..61b6cdf16 100644 --- a/kubernetes/loculus/templates/_common-metadata.tpl +++ b/kubernetes/loculus/templates/_common-metadata.tpl @@ -107,6 +107,10 @@ fields: type: string displayName: Version comment header: Submission details + - name: pipelineVersion + type: int + notSearchable: true + hideOnSequenceDetailsPage: true {{- end}} {{/* Patches schema by adding to it and overwriting overlapping fields by the value in metadataAdd*/}}