From 651be1187c7136fba6d52c046bbda8a2a0a19697 Mon Sep 17 00:00:00 2001 From: Tobias Kampmann Date: Wed, 17 Jan 2024 18:29:53 +0100 Subject: [PATCH] feat(backend): implement endpoint to change data use terms of sequence entries #763 --- .../org/loculus/backend/api/DataUseTerms.kt | 72 ++++----- .../loculus/backend/api/SubmissionTypes.kt | 4 + .../backend/config/BackendSpringConfig.kt | 2 +- .../controller/DataUseTermsController.kt | 19 +-- .../controller/SubmissionController.kt | 30 ++-- .../backend/model/ReleasedDataModel.kt | 10 +- .../org/loculus/backend/model/SubmitModel.kt | 3 + .../DataUseTermsDatabaseService.kt | 36 ++--- .../DataUseTermsPreconditionValidator.kt | 77 ++++++++++ .../service/datauseterms/DataUseTermsTable.kt | 35 +++++ .../datauseterms/DataUseTermsTables.kt | 16 -- ...r.kt => AccessionPreconditionValidator.kt} | 41 ++++- .../submission/SequenceEntriesTable.kt | 115 +++++++++----- ...ervice.kt => SubmissionDatabaseService.kt} | 16 +- .../submission/UploadDatabaseService.kt | 6 +- .../backend/utils/AccessionComparators.kt | 4 +- .../{service => api}/DataUseTermsTest.kt | 11 +- .../controller/EndpointTestExtension.kt | 6 +- .../DataUseTermsControllerClient.kt | 30 ++++ .../DataUseTermsControllerTest.kt | 140 ++++++++++++++++++ .../submission/GetDataToEditEndpointTest.kt | 5 +- .../submission/ReviseEndpointTest.kt | 5 +- .../submission/SubmissionControllerClient.kt | 15 +- .../submission/SubmissionConvenienceClient.kt | 3 + .../submission/SubmitEndpointTest.kt | 59 +++----- 25 files changed, 540 insertions(+), 220 deletions(-) create mode 100644 backend/src/main/kotlin/org/loculus/backend/service/datauseterms/DataUseTermsPreconditionValidator.kt create mode 100644 backend/src/main/kotlin/org/loculus/backend/service/datauseterms/DataUseTermsTable.kt delete mode 100644 backend/src/main/kotlin/org/loculus/backend/service/datauseterms/DataUseTermsTables.kt rename backend/src/main/kotlin/org/loculus/backend/service/submission/{SubmissionPreconditionValidator.kt => AccessionPreconditionValidator.kt} (86%) rename backend/src/main/kotlin/org/loculus/backend/service/submission/{DatabaseService.kt => SubmissionDatabaseService.kt} (97%) rename backend/src/test/kotlin/org/loculus/backend/{service => api}/DataUseTermsTest.kt (89%) create mode 100644 backend/src/test/kotlin/org/loculus/backend/controller/datauseterms/DataUseTermsControllerClient.kt create mode 100644 backend/src/test/kotlin/org/loculus/backend/controller/datauseterms/DataUseTermsControllerTest.kt diff --git a/backend/src/main/kotlin/org/loculus/backend/api/DataUseTerms.kt b/backend/src/main/kotlin/org/loculus/backend/api/DataUseTerms.kt index f0c1e0b918..e50fd15ae1 100644 --- a/backend/src/main/kotlin/org/loculus/backend/api/DataUseTerms.kt +++ b/backend/src/main/kotlin/org/loculus/backend/api/DataUseTerms.kt @@ -1,6 +1,7 @@ package org.loculus.backend.api import com.fasterxml.jackson.annotation.JsonIgnore +import com.fasterxml.jackson.annotation.JsonProperty import com.fasterxml.jackson.annotation.JsonPropertyOrder import com.fasterxml.jackson.annotation.JsonSubTypes import com.fasterxml.jackson.annotation.JsonTypeInfo @@ -9,43 +10,60 @@ import com.fasterxml.jackson.core.JsonGenerator import com.fasterxml.jackson.databind.SerializerProvider import com.fasterxml.jackson.databind.annotation.JsonSerialize import com.fasterxml.jackson.databind.ser.std.StdSerializer -import kotlinx.datetime.Clock -import kotlinx.datetime.DateTimeUnit.Companion.YEAR +import io.swagger.v3.oas.annotations.media.Schema import kotlinx.datetime.LocalDate import kotlinx.datetime.LocalDateTime -import kotlinx.datetime.TimeZone -import kotlinx.datetime.plus -import kotlinx.datetime.toLocalDateTime import mu.KotlinLogging -import org.loculus.backend.config.logger import org.loculus.backend.controller.BadRequestException +import org.loculus.backend.utils.Accession enum class DataUseTermsType { + @JsonProperty("OPEN") OPEN, + + @JsonProperty("RESTRICTED") RESTRICTED, + + ; + + companion object { + private val stringToEnumMap: Map = entries.associateBy { it.name } + + fun fromString(dataUseTermsTypeString: String): DataUseTermsType { + return stringToEnumMap[dataUseTermsTypeString] + ?: throw IllegalArgumentException("Unknown DataUseTermsType: $dataUseTermsTypeString") + } + } } -val logger = KotlinLogging.logger { } +private val logger = KotlinLogging.logger { } @JsonTypeInfo(use = JsonTypeInfo.Id.NAME, include = JsonTypeInfo.As.PROPERTY, property = "type") @JsonSubTypes( JsonSubTypes.Type(value = DataUseTerms.Open::class, name = "OPEN"), JsonSubTypes.Type(value = DataUseTerms.Restricted::class, name = "RESTRICTED"), ) -@JsonPropertyOrder(value = ["type", "restrictedUntil", "changeDateTime"]) +@JsonPropertyOrder(value = ["type", "restrictedUntil"]) sealed interface DataUseTerms { val type: DataUseTermsType @JsonTypeName("OPEN") - data class Open(private val dummy: String = "") : - DataUseTerms { + @Schema(description = "The sequence entry is open access. No restrictions apply.") + data object Open : DataUseTerms { @JsonIgnore override val type = DataUseTermsType.OPEN } @JsonTypeName("RESTRICTED") + @Schema(description = "The sequence entry is restricted access.") data class Restricted( @JsonSerialize(using = LocalDateSerializer::class) + @Schema( + description = "The date (YYYY-MM-DD) until which the sequence entry is restricted.", + type = "string", + format = "date", + example = "2021-01-01", + ) val restrictedUntil: LocalDate, ) : DataUseTerms { @JsonIgnore @@ -56,12 +74,8 @@ sealed interface DataUseTerms { fun fromParameters(type: DataUseTermsType, restrictedUntilString: String?): DataUseTerms { logger.info { "Creating DataUseTerms from parameters: type=$type, restrictedUntil=$restrictedUntilString" } return when (type) { - DataUseTermsType.OPEN -> Open() - DataUseTermsType.RESTRICTED -> { - val restrictedUntil = parseRestrictedUntil(restrictedUntilString) - validateRestrictedUntil(restrictedUntil) - Restricted(restrictedUntil) - } + DataUseTermsType.OPEN -> Open + DataUseTermsType.RESTRICTED -> Restricted(parseRestrictedUntil(restrictedUntilString)) } } @@ -77,32 +91,22 @@ sealed interface DataUseTerms { ) } } - - private fun validateRestrictedUntil(restrictedUntil: LocalDate) { - val now = Clock.System.now().toLocalDateTime(TimeZone.UTC).date - val oneYearFromNow = now.plus(1, YEAR) - - if (restrictedUntil < now) { - throw BadRequestException( - "The date 'restrictedUntil' must be in the future, up to a maximum of 1 year from now.", - ) - } - if (restrictedUntil > oneYearFromNow) { - throw BadRequestException( - "The date 'restrictedUntil' must not exceed 1 year from today.", - ) - } - } } } -class LocalDateSerializer : StdSerializer(LocalDate::class.java) { +data class DataUseTermsChangeRequest( + @Schema(description = "A list of accessions of the dataset to set the data use terms for") + val accessions: List, + val newDataUseTerms: DataUseTerms, +) + +private class LocalDateSerializer : StdSerializer(LocalDate::class.java) { override fun serialize(value: LocalDate, gen: JsonGenerator, provider: SerializerProvider) { gen.writeString(value.toString()) } } -class LocalDateTimeSerializer : StdSerializer(LocalDateTime::class.java) { +private class LocalDateTimeSerializer : StdSerializer(LocalDateTime::class.java) { override fun serialize(value: LocalDateTime, gen: JsonGenerator, provider: SerializerProvider) { gen.writeString(value.toString()) } diff --git a/backend/src/main/kotlin/org/loculus/backend/api/SubmissionTypes.kt b/backend/src/main/kotlin/org/loculus/backend/api/SubmissionTypes.kt index 98618f3139..9ba43cec9d 100644 --- a/backend/src/main/kotlin/org/loculus/backend/api/SubmissionTypes.kt +++ b/backend/src/main/kotlin/org/loculus/backend/api/SubmissionTypes.kt @@ -11,6 +11,10 @@ import io.swagger.v3.oas.annotations.media.Schema import org.loculus.backend.utils.Accession import org.loculus.backend.utils.Version +data class Accessions( + val accessions: List, +) + interface AccessionVersionInterface { val accession: Accession val version: Version diff --git a/backend/src/main/kotlin/org/loculus/backend/config/BackendSpringConfig.kt b/backend/src/main/kotlin/org/loculus/backend/config/BackendSpringConfig.kt index 1dc3d6dad2..b09c87f92b 100644 --- a/backend/src/main/kotlin/org/loculus/backend/config/BackendSpringConfig.kt +++ b/backend/src/main/kotlin/org/loculus/backend/config/BackendSpringConfig.kt @@ -20,7 +20,7 @@ object BackendSpringProperty { const val BACKEND_CONFIG_PATH = "backend.config.path" } -val logger = mu.KotlinLogging.logger {} +private val logger = mu.KotlinLogging.logger {} @Configuration @ImportAutoConfiguration( diff --git a/backend/src/main/kotlin/org/loculus/backend/controller/DataUseTermsController.kt b/backend/src/main/kotlin/org/loculus/backend/controller/DataUseTermsController.kt index 0ad0e66f8c..d01588e94d 100644 --- a/backend/src/main/kotlin/org/loculus/backend/controller/DataUseTermsController.kt +++ b/backend/src/main/kotlin/org/loculus/backend/controller/DataUseTermsController.kt @@ -3,14 +3,12 @@ package org.loculus.backend.controller import io.swagger.v3.oas.annotations.Operation import io.swagger.v3.oas.annotations.Parameter import io.swagger.v3.oas.annotations.security.SecurityRequirement -import org.loculus.backend.api.DataUseTerms +import org.loculus.backend.api.DataUseTermsChangeRequest import org.loculus.backend.service.datauseterms.DataUseTermsDatabaseService -import org.loculus.backend.utils.Accession import org.springframework.http.HttpStatus import org.springframework.http.MediaType import org.springframework.web.bind.annotation.PutMapping import org.springframework.web.bind.annotation.RequestBody -import org.springframework.web.bind.annotation.RequestParam import org.springframework.web.bind.annotation.ResponseStatus import org.springframework.web.bind.annotation.RestController @@ -20,20 +18,17 @@ class DataUseTermsController( private val dataUseTermsDatabaseService: DataUseTermsDatabaseService, ) { - @Operation(description = "Set new data use terms. Until now, just testing purposes") + @Operation( + description = "Change the data use terms of the given accessions. Only a change to more open terms is allowed.", + ) @ResponseStatus(HttpStatus.NO_CONTENT) @PutMapping("/data-use-terms", produces = [MediaType.APPLICATION_JSON_VALUE]) fun setNewDataUseTerms( @UsernameFromJwt username: String, - @Parameter( - description = "The accession of the dataset to set the data use terms for", - ) @RequestParam accession: Accession, - @Parameter( - description = "The new data use terms", - ) @RequestBody newDataUseTerms: DataUseTerms, + @Parameter @RequestBody request: DataUseTermsChangeRequest, ) = dataUseTermsDatabaseService.setNewDataUseTerms( - listOf(accession), username, - DataUseTerms.Open(), + request.accessions, + request.newDataUseTerms, ) } diff --git a/backend/src/main/kotlin/org/loculus/backend/controller/SubmissionController.kt b/backend/src/main/kotlin/org/loculus/backend/controller/SubmissionController.kt index 72ca5d76bf..4fc0a58c2d 100644 --- a/backend/src/main/kotlin/org/loculus/backend/controller/SubmissionController.kt +++ b/backend/src/main/kotlin/org/loculus/backend/controller/SubmissionController.kt @@ -11,6 +11,7 @@ import jakarta.validation.Valid import jakarta.validation.constraints.Max import mu.KotlinLogging import org.loculus.backend.api.AccessionVersion +import org.loculus.backend.api.Accessions import org.loculus.backend.api.DataUseTerms import org.loculus.backend.api.DataUseTermsType import org.loculus.backend.api.Organism @@ -23,7 +24,7 @@ import org.loculus.backend.api.UnprocessedData import org.loculus.backend.model.ReleasedDataModel import org.loculus.backend.model.SubmissionParams import org.loculus.backend.model.SubmitModel -import org.loculus.backend.service.submission.DatabaseService +import org.loculus.backend.service.submission.SubmissionDatabaseService import org.loculus.backend.utils.Accession import org.loculus.backend.utils.IteratorStreamer import org.springframework.http.HttpHeaders @@ -55,7 +56,7 @@ private val log = KotlinLogging.logger { } class SubmissionController( private val submitModel: SubmitModel, private val releasedDataModel: ReleasedDataModel, - private val databaseService: DatabaseService, + private val submissionDatabaseService: SubmissionDatabaseService, private val iteratorStreamer: IteratorStreamer, ) { @@ -136,7 +137,8 @@ class SubmissionController( val headers = HttpHeaders() headers.contentType = MediaType.parseMediaType(MediaType.APPLICATION_NDJSON_VALUE) - val streamBody = stream { databaseService.streamUnprocessedSubmissions(numberOfSequenceEntries, organism) } + val streamBody = + stream { submissionDatabaseService.streamUnprocessedSubmissions(numberOfSequenceEntries, organism) } return ResponseEntity(streamBody, headers, HttpStatus.OK) } @@ -159,7 +161,7 @@ class SubmissionController( @PathVariable @Valid organism: Organism, request: HttpServletRequest, - ) = databaseService.updateProcessedData(request.inputStream, organism) + ) = submissionDatabaseService.updateProcessedData(request.inputStream, organism) @Operation(description = GET_RELEASED_DATA_DESCRIPTION) @ResponseStatus(HttpStatus.OK) @@ -203,7 +205,7 @@ class SubmissionController( val headers = HttpHeaders() headers.contentType = MediaType.parseMediaType(MediaType.APPLICATION_NDJSON_VALUE) - val entries = databaseService.streamDataToEdit(username, groupName, numberOfSequenceEntries, organism) + val entries = submissionDatabaseService.streamDataToEdit(username, groupName, numberOfSequenceEntries, organism) val streamBody = stream { entries } return ResponseEntity(streamBody, headers, HttpStatus.OK) @@ -217,7 +219,7 @@ class SubmissionController( @PathVariable accession: Accession, @PathVariable version: Long, @UsernameFromJwt username: String, - ): SequenceEntryVersionToEdit = databaseService.getSequenceEntryVersionToEdit( + ): SequenceEntryVersionToEdit = submissionDatabaseService.getSequenceEntryVersionToEdit( username, AccessionVersion(accession, version), organism, @@ -231,7 +233,7 @@ class SubmissionController( organism: Organism, @UsernameFromJwt username: String, @RequestBody accessionVersion: UnprocessedData, - ) = databaseService.submitEditedData(username, accessionVersion, organism) + ) = submissionDatabaseService.submitEditedData(username, accessionVersion, organism) @Operation(description = GET_SEQUENCES_OF_USER_DESCRIPTION) @GetMapping("/get-sequences-of-user", produces = [MediaType.APPLICATION_JSON_VALUE]) @@ -240,7 +242,7 @@ class SubmissionController( organism: Organism, @UsernameFromJwt username: String, ): List { - return databaseService.getActiveSequencesSubmittedBy(username, organism) + return submissionDatabaseService.getActiveSequencesSubmittedBy(username, organism) } @Operation(description = APPROVE_PROCESSED_DATA_DESCRIPTION) @@ -252,7 +254,7 @@ class SubmissionController( @UsernameFromJwt username: String, @RequestBody body: AccessionVersions, ) { - databaseService.approveProcessedData(username, body.accessionVersions, organism) + submissionDatabaseService.approveProcessedData(username, body.accessionVersions, organism) } @Operation(description = REVOKE_DESCRIPTION) @@ -262,7 +264,7 @@ class SubmissionController( organism: Organism, @RequestBody body: Accessions, @UsernameFromJwt username: String, - ): List = databaseService.revoke(body.accessions, username, organism) + ): List = submissionDatabaseService.revoke(body.accessions, username, organism) @Operation(description = CONFIRM_REVOCATION_DESCRIPTION) @ResponseStatus(HttpStatus.NO_CONTENT) @@ -272,7 +274,7 @@ class SubmissionController( organism: Organism, @UsernameFromJwt username: String, @RequestBody body: AccessionVersions, - ) = databaseService.confirmRevocation(body.accessionVersions, username, organism) + ) = submissionDatabaseService.confirmRevocation(body.accessionVersions, username, organism) @Operation(description = DELETE_SEQUENCES_DESCRIPTION) @ResponseStatus(HttpStatus.NO_CONTENT) @@ -284,7 +286,7 @@ class SubmissionController( organism: Organism, @UsernameFromJwt username: String, @RequestBody body: AccessionVersions, - ) = databaseService.deleteSequenceEntryVersions(body.accessionVersions, username, organism) + ) = submissionDatabaseService.deleteSequenceEntryVersions(body.accessionVersions, username, organism) private fun stream(sequenceProvider: () -> Sequence) = StreamingResponseBody { outputStream -> try { @@ -297,10 +299,6 @@ class SubmissionController( } } - data class Accessions( - val accessions: List, - ) - data class AccessionVersions( val accessionVersions: List, ) diff --git a/backend/src/main/kotlin/org/loculus/backend/model/ReleasedDataModel.kt b/backend/src/main/kotlin/org/loculus/backend/model/ReleasedDataModel.kt index f4f5d8250c..8a1e9664c0 100644 --- a/backend/src/main/kotlin/org/loculus/backend/model/ReleasedDataModel.kt +++ b/backend/src/main/kotlin/org/loculus/backend/model/ReleasedDataModel.kt @@ -6,8 +6,8 @@ import mu.KotlinLogging import org.loculus.backend.api.Organism import org.loculus.backend.api.ProcessedData import org.loculus.backend.api.SiloVersionStatus -import org.loculus.backend.service.submission.DatabaseService import org.loculus.backend.service.submission.RawProcessedData +import org.loculus.backend.service.submission.SubmissionDatabaseService import org.loculus.backend.utils.Accession import org.loculus.backend.utils.Version import org.springframework.stereotype.Service @@ -16,15 +16,15 @@ import org.springframework.transaction.annotation.Transactional private val log = KotlinLogging.logger { } @Service -class ReleasedDataModel(private val databaseService: DatabaseService) { +class ReleasedDataModel(private val submissionDatabaseService: SubmissionDatabaseService) { @Transactional(readOnly = true) fun getReleasedData(organism: Organism): Sequence { log.info { "fetching released submissions" } - val latestVersions = databaseService.getLatestVersions(organism) - val latestRevocationVersions = databaseService.getLatestRevocationVersions(organism) + val latestVersions = submissionDatabaseService.getLatestVersions(organism) + val latestRevocationVersions = submissionDatabaseService.getLatestRevocationVersions(organism) - return databaseService.streamReleasedSubmissions(organism) + return submissionDatabaseService.streamReleasedSubmissions(organism) .map { computeAdditionalMetadataFields(it, latestVersions, latestRevocationVersions) } } diff --git a/backend/src/main/kotlin/org/loculus/backend/model/SubmitModel.kt b/backend/src/main/kotlin/org/loculus/backend/model/SubmitModel.kt index 6c8aa995a0..70d3e5d019 100644 --- a/backend/src/main/kotlin/org/loculus/backend/model/SubmitModel.kt +++ b/backend/src/main/kotlin/org/loculus/backend/model/SubmitModel.kt @@ -13,6 +13,7 @@ import org.loculus.backend.api.SubmissionIdMapping import org.loculus.backend.controller.BadRequestException import org.loculus.backend.controller.DuplicateKeyException import org.loculus.backend.controller.UnprocessableEntityException +import org.loculus.backend.service.datauseterms.DataUseTermsPreconditionValidator import org.loculus.backend.service.groupmanagement.GroupManagementPreconditionValidator import org.loculus.backend.service.submission.CompressionAlgorithm import org.loculus.backend.service.submission.UploadDatabaseService @@ -70,6 +71,7 @@ enum class UploadType { class SubmitModel( private val uploadDatabaseService: UploadDatabaseService, private val groupManagementPreconditionValidator: GroupManagementPreconditionValidator, + private val dataUseTermsPreconditionValidator: DataUseTermsPreconditionValidator, ) { companion object AcceptedFileTypes { @@ -130,6 +132,7 @@ class SubmitModel( submissionParams.groupName, submissionParams.username, ) + dataUseTermsPreconditionValidator.checkThatRestrictedUntilIsAllowed(submissionParams.dataUseTerms) } val metadataTempFileToDelete = MaybeFile() diff --git a/backend/src/main/kotlin/org/loculus/backend/service/datauseterms/DataUseTermsDatabaseService.kt b/backend/src/main/kotlin/org/loculus/backend/service/datauseterms/DataUseTermsDatabaseService.kt index 495150ab3c..ca7976b662 100644 --- a/backend/src/main/kotlin/org/loculus/backend/service/datauseterms/DataUseTermsDatabaseService.kt +++ b/backend/src/main/kotlin/org/loculus/backend/service/datauseterms/DataUseTermsDatabaseService.kt @@ -3,38 +3,38 @@ package org.loculus.backend.service.datauseterms import kotlinx.datetime.Clock import kotlinx.datetime.TimeZone import kotlinx.datetime.toLocalDateTime -import mu.KotlinLogging import org.jetbrains.exposed.sql.batchInsert import org.loculus.backend.api.DataUseTerms +import org.loculus.backend.service.submission.AccessionPreconditionValidator +import org.loculus.backend.utils.Accession import org.springframework.stereotype.Service import org.springframework.transaction.annotation.Transactional -private val log = KotlinLogging.logger { } - @Service @Transactional -class DataUseTermsDatabaseService { +class DataUseTermsDatabaseService( + private val accessionPreconditionValidator: AccessionPreconditionValidator, + private val dataUseTermsPreconditionValidator: DataUseTermsPreconditionValidator, +) { - fun setNewDataUseTerms(accessions: List, username: String, newDataUseTerms: DataUseTerms) { - log.info { - "Setting new data use terms for accessions $accessions. " + - "Just an entry in the new Table. " + - "Will be filled with real juicy logic in the next tickets. See #760 ff. " - } + fun setNewDataUseTerms(username: String, accessions: List, newDataUseTerms: DataUseTerms) { val now = Clock.System.now().toLocalDateTime(TimeZone.UTC) + accessionPreconditionValidator.validateAccessions( + submitter = username, + accessions = accessions, + ) + + dataUseTermsPreconditionValidator.checkThatTransitionIsAllowed(accessions, newDataUseTerms) + dataUseTermsPreconditionValidator.checkThatRestrictedUntilIsAllowed(newDataUseTerms) + DataUseTermsTable.batchInsert(accessions) { this[DataUseTermsTable.accessionColumn] = it this[DataUseTermsTable.changeDateColumn] = now - this[DataUseTermsTable.dataUseTermsTypeColumn] = newDataUseTerms.type + this[DataUseTermsTable.dataUseTermsTypeColumn] = newDataUseTerms.type.toString() this[DataUseTermsTable.restrictedUntilColumn] = when (newDataUseTerms) { - is DataUseTerms.Restricted -> { - newDataUseTerms.restrictedUntil - } - - else -> { - null - } + is DataUseTerms.Restricted -> newDataUseTerms.restrictedUntil + else -> null } this[DataUseTermsTable.userNameColumn] = username } diff --git a/backend/src/main/kotlin/org/loculus/backend/service/datauseterms/DataUseTermsPreconditionValidator.kt b/backend/src/main/kotlin/org/loculus/backend/service/datauseterms/DataUseTermsPreconditionValidator.kt new file mode 100644 index 0000000000..2b76484da1 --- /dev/null +++ b/backend/src/main/kotlin/org/loculus/backend/service/datauseterms/DataUseTermsPreconditionValidator.kt @@ -0,0 +1,77 @@ +package org.loculus.backend.service.datauseterms + +import kotlinx.datetime.Clock +import kotlinx.datetime.DateTimeUnit +import kotlinx.datetime.TimeZone +import kotlinx.datetime.plus +import kotlinx.datetime.toLocalDateTime +import mu.KotlinLogging +import org.jetbrains.exposed.sql.and +import org.jetbrains.exposed.sql.select +import org.loculus.backend.api.DataUseTerms +import org.loculus.backend.api.DataUseTermsType +import org.loculus.backend.controller.BadRequestException +import org.loculus.backend.controller.UnprocessableEntityException +import org.loculus.backend.utils.Accession +import org.springframework.stereotype.Component + +private val logger = KotlinLogging.logger { } + +@Component +class DataUseTermsPreconditionValidator { + + fun checkThatTransitionIsAllowed(accessions: List, newDataUseTerms: DataUseTerms) { + val dataUseTerms = DataUseTermsTable + .slice( + DataUseTermsTable.accessionColumn, + DataUseTermsTable.dataUseTermsTypeColumn, + DataUseTermsTable.restrictedUntilColumn, + ) + .select( + where = { + (DataUseTermsTable.accessionColumn inList accessions) and DataUseTermsTable.isNewestDataUseTerms + }, + ) + + logger.debug { + "Checking that transition is allowed for accessions " + + "$accessions and new data use terms $newDataUseTerms. Found $dataUseTerms." + } + + if (newDataUseTerms is DataUseTerms.Restricted) { + dataUseTerms.forEach { + val dataUseTermsType = DataUseTermsType.fromString(it[DataUseTermsTable.dataUseTermsTypeColumn]) + if (dataUseTermsType == DataUseTermsType.OPEN) { + throw UnprocessableEntityException("Cannot change data use terms from OPEN to RESTRICTED.") + } + + val oldRestrictedUntilDate = it[DataUseTermsTable.restrictedUntilColumn] + ?: throw RuntimeException("Data use terms are RESTRICTED but restrictedUntil is null. Aborting.") + if (oldRestrictedUntilDate < newDataUseTerms.restrictedUntil) { + throw UnprocessableEntityException( + "Cannot extend restricted data use period. Please choose a date before " + + "$oldRestrictedUntilDate.", + ) + } + } + } + } + + fun checkThatRestrictedUntilIsAllowed(dataUseTerms: DataUseTerms) { + if (dataUseTerms is DataUseTerms.Restricted) { + val now = Clock.System.now().toLocalDateTime(TimeZone.UTC).date + val oneYearFromNow = now.plus(1, DateTimeUnit.YEAR) + + if (dataUseTerms.restrictedUntil < now) { + throw BadRequestException( + "The date 'restrictedUntil' must be in the future, up to a maximum of 1 year from now.", + ) + } + if (dataUseTerms.restrictedUntil > oneYearFromNow) { + throw BadRequestException( + "The date 'restrictedUntil' must not exceed 1 year from today.", + ) + } + } + } +} diff --git a/backend/src/main/kotlin/org/loculus/backend/service/datauseterms/DataUseTermsTable.kt b/backend/src/main/kotlin/org/loculus/backend/service/datauseterms/DataUseTermsTable.kt new file mode 100644 index 0000000000..724d736cb6 --- /dev/null +++ b/backend/src/main/kotlin/org/loculus/backend/service/datauseterms/DataUseTermsTable.kt @@ -0,0 +1,35 @@ +package org.loculus.backend.service.datauseterms + +import kotlinx.datetime.LocalDateTime +import org.jetbrains.exposed.sql.Expression +import org.jetbrains.exposed.sql.SqlExpressionBuilder.eq +import org.jetbrains.exposed.sql.Table +import org.jetbrains.exposed.sql.alias +import org.jetbrains.exposed.sql.kotlin.datetime.date +import org.jetbrains.exposed.sql.kotlin.datetime.datetime +import org.jetbrains.exposed.sql.max +import org.jetbrains.exposed.sql.select +import org.jetbrains.exposed.sql.wrapAsExpression + +const val DATA_USE_TERMS_TABLE_NAME = "data_use_terms_table" + +object DataUseTermsTable : Table(DATA_USE_TERMS_TABLE_NAME) { + val accessionColumn = text("accession") + val changeDateColumn = datetime("change_date") + val dataUseTermsTypeColumn = text("data_use_terms_type") + val restrictedUntilColumn = date("restricted_until").nullable() + val userNameColumn = text("user_name") + + val isNewestDataUseTerms = changeDateColumn eq newestDataUseTermsQuery() + + private fun newestDataUseTermsQuery(): Expression { + val subQueryTable = alias("subQueryTable") + return wrapAsExpression( + subQueryTable + .slice(subQueryTable[changeDateColumn].max()) + .select { + subQueryTable[accessionColumn] eq accessionColumn + }, + ) + } +} diff --git a/backend/src/main/kotlin/org/loculus/backend/service/datauseterms/DataUseTermsTables.kt b/backend/src/main/kotlin/org/loculus/backend/service/datauseterms/DataUseTermsTables.kt deleted file mode 100644 index 53ed0b3548..0000000000 --- a/backend/src/main/kotlin/org/loculus/backend/service/datauseterms/DataUseTermsTables.kt +++ /dev/null @@ -1,16 +0,0 @@ -package org.loculus.backend.service.datauseterms - -import org.jetbrains.exposed.sql.Table -import org.jetbrains.exposed.sql.kotlin.datetime.date -import org.jetbrains.exposed.sql.kotlin.datetime.datetime -import org.loculus.backend.api.DataUseTermsType - -const val DATA_USE_TERMS_TABLE_NAME = "data_use_terms_table" - -object DataUseTermsTable : Table(DATA_USE_TERMS_TABLE_NAME) { - val accessionColumn = text("accession") - val changeDateColumn = datetime("change_date") - val dataUseTermsTypeColumn = enumeration("data_use_terms_type", DataUseTermsType::class) - val restrictedUntilColumn = date("restricted_until").nullable() - val userNameColumn = text("user_name") -} diff --git a/backend/src/main/kotlin/org/loculus/backend/service/submission/SubmissionPreconditionValidator.kt b/backend/src/main/kotlin/org/loculus/backend/service/submission/AccessionPreconditionValidator.kt similarity index 86% rename from backend/src/main/kotlin/org/loculus/backend/service/submission/SubmissionPreconditionValidator.kt rename to backend/src/main/kotlin/org/loculus/backend/service/submission/AccessionPreconditionValidator.kt index eeb301d47b..58afb72173 100644 --- a/backend/src/main/kotlin/org/loculus/backend/service/submission/SubmissionPreconditionValidator.kt +++ b/backend/src/main/kotlin/org/loculus/backend/service/submission/AccessionPreconditionValidator.kt @@ -17,7 +17,7 @@ import org.loculus.backend.utils.Version import org.springframework.stereotype.Component @Component -class SubmissionPreconditionValidator( +class AccessionPreconditionValidator( private val sequenceEntriesTableProvider: SequenceEntriesTableProvider, private val groupManagementPreconditionValidator: GroupManagementPreconditionValidator, ) { @@ -82,6 +82,32 @@ class SubmissionPreconditionValidator( } } + fun validateAccessions(submitter: String, accessions: List): List { + sequenceEntriesTableProvider.get(organism = null).let { table -> + val sequenceEntries = table + .slice( + table.accessionColumn, + table.versionColumn, + table.submitterColumn, + table.groupNameColumn, + ) + .select( + where = { (table.accessionColumn inList accessions) and table.isMaxVersion }, + ) + + validateAccessionsExist(sequenceEntries, accessions, table) + validateUserIsAllowedToEditSequenceEntries(sequenceEntries, submitter, table) + + return sequenceEntries.map { + AccessionVersionGroup( + it[table.accessionColumn], + it[table.versionColumn], + it[table.groupNameColumn], + ) + } + } + } + private fun validateAccessionVersionsExist( sequenceEntries: Query, accessionVersions: List, @@ -138,11 +164,14 @@ class SubmissionPreconditionValidator( table: SequenceEntriesDataTable, ) { val groupsOfSequenceEntries = sequenceEntries - .groupBy({ - it[table.groupNameColumn] - }, { - AccessionVersion(it[table.accessionColumn], it[table.versionColumn]) - }) + .groupBy( + { + it[table.groupNameColumn] + }, + { + AccessionVersion(it[table.accessionColumn], it[table.versionColumn]) + }, + ) groupsOfSequenceEntries.forEach { (groupName, accessionList) -> try { diff --git a/backend/src/main/kotlin/org/loculus/backend/service/submission/SequenceEntriesTable.kt b/backend/src/main/kotlin/org/loculus/backend/service/submission/SequenceEntriesTable.kt index a9ca68c3da..4b921cf18e 100644 --- a/backend/src/main/kotlin/org/loculus/backend/service/submission/SequenceEntriesTable.kt +++ b/backend/src/main/kotlin/org/loculus/backend/service/submission/SequenceEntriesTable.kt @@ -1,6 +1,7 @@ package org.loculus.backend.service.submission import com.fasterxml.jackson.module.kotlin.readValue +import mu.KotlinLogging import org.jetbrains.exposed.sql.Column import org.jetbrains.exposed.sql.Expression import org.jetbrains.exposed.sql.SqlExpressionBuilder.eq @@ -24,12 +25,14 @@ import org.loculus.backend.service.jacksonObjectMapper import org.loculus.backend.service.jacksonSerializableJsonb import org.springframework.stereotype.Service +private val logger = KotlinLogging.logger { } + @Service class SequenceEntriesTableProvider(private val compressionService: CompressionService) { - private val cachedTables: MutableMap = mutableMapOf() + private val cachedTables: MutableMap = mutableMapOf() - fun get(organism: Organism): SequenceEntriesDataTable { + fun get(organism: Organism?): SequenceEntriesDataTable { return cachedTables.getOrPut(organism) { SequenceEntriesDataTable(compressionService, organism) } @@ -40,7 +43,7 @@ const val SEQUENCE_ENTRIES_TABLE_NAME = "sequence_entries" class SequenceEntriesDataTable( compressionService: CompressionService, - organism: Organism, + organism: Organism? = null, ) : Table( SEQUENCE_ENTRIES_TABLE_NAME, ) { @@ -98,53 +101,83 @@ class SequenceEntriesDataTable( fun statusIs(status: Status) = statusColumn eq status.name - fun statusIsOneOf(vararg status: Status) = statusColumn inList status.map { it.name } - fun accessionVersionEquals(accessionVersion: AccessionVersionInterface) = (accessionColumn eq accessionVersion.accession) and (versionColumn eq accessionVersion.version) fun groupIs(group: String) = groupNameColumn eq group + private val warningWhenNoOrganismWhenSerializing = "Organism is null when de-serializing data. " + + "This should not happen. " + + "Please check your code. " + + "Data will be written without compression. " + + "If this is unintentional data can become corrupted. " + private fun serializeOriginalData( compressionService: CompressionService, - organism: Organism, - ): Column = jsonb( - "original_data", - { originalData -> - jacksonObjectMapper.writeValueAsString( - compressionService.compressSequencesInOriginalData( - originalData, - organism, - ), - ) - }, - { string -> - compressionService.decompressSequencesInOriginalData( - jacksonObjectMapper.readValue(string) as OriginalData, - organism, - ) - }, - ) + organism: Organism?, + ): Column { + return jsonb( + "original_data", + { originalData -> + jacksonObjectMapper.writeValueAsString( + if (organism == null) { + logger.warn { warningWhenNoOrganismWhenSerializing } + originalData + } else { + compressionService.compressSequencesInOriginalData( + originalData, + organism, + ) + }, + ) + }, + { string -> + val originalData = jacksonObjectMapper.readValue(string) as OriginalData + if (organism == null) { + logger.warn { warningWhenNoOrganismWhenSerializing } + originalData + } else { + compressionService.decompressSequencesInOriginalData( + originalData, + organism, + ) + } + }, + ) + } private fun serializeProcessedData( compressionService: CompressionService, - organism: Organism, - ): Column = jsonb( - "processed_data", - { processedData -> - jacksonObjectMapper.writeValueAsString( - compressionService.compressSequencesInProcessedData( - processedData, - organism, - ), - ) - }, - { string -> - compressionService.decompressSequencesInProcessedData( - jacksonObjectMapper.readValue(string) as ProcessedData, - organism, - ) - }, - ) + organism: Organism?, + ): Column { + return jsonb( + "processed_data", + { processedData -> + jacksonObjectMapper.writeValueAsString( + if (organism == null) { + logger.warn { warningWhenNoOrganismWhenSerializing } + processedData + } else { + compressionService.compressSequencesInProcessedData( + processedData, + organism, + ) + }, + ) + }, + { string -> + val processedData = jacksonObjectMapper.readValue(string) as ProcessedData + if (organism == null) { + logger.warn { warningWhenNoOrganismWhenSerializing } + processedData + } else { + compressionService.decompressSequencesInProcessedData( + jacksonObjectMapper.readValue(string) as ProcessedData, + organism, + ) + } + }, + ) + } } diff --git a/backend/src/main/kotlin/org/loculus/backend/service/submission/DatabaseService.kt b/backend/src/main/kotlin/org/loculus/backend/service/submission/SubmissionDatabaseService.kt similarity index 97% rename from backend/src/main/kotlin/org/loculus/backend/service/submission/DatabaseService.kt rename to backend/src/main/kotlin/org/loculus/backend/service/submission/SubmissionDatabaseService.kt index cdb53cc070..80665571ce 100644 --- a/backend/src/main/kotlin/org/loculus/backend/service/submission/DatabaseService.kt +++ b/backend/src/main/kotlin/org/loculus/backend/service/submission/SubmissionDatabaseService.kt @@ -52,9 +52,9 @@ private val log = KotlinLogging.logger { } @Service @Transactional -class DatabaseService( +class SubmissionDatabaseService( private val processedSequenceEntryValidatorFactory: ProcessedSequenceEntryValidatorFactory, - private val submissionPreconditionValidator: SubmissionPreconditionValidator, + private val accessionPreconditionValidator: AccessionPreconditionValidator, private val groupManagementPreconditionValidator: GroupManagementPreconditionValidator, private val objectMapper: ObjectMapper, pool: DataSource, @@ -211,7 +211,7 @@ class DatabaseService( val now = Clock.System.now().toLocalDateTime(TimeZone.UTC) - submissionPreconditionValidator.validateAccessionVersions( + accessionPreconditionValidator.validateAccessionVersions( submitter, accessionVersions, listOf(AWAITING_APPROVAL), @@ -388,7 +388,7 @@ class DatabaseService( fun revoke(accessions: List, username: String, organism: Organism): List { log.info { "revoking ${accessions.size} sequences" } - submissionPreconditionValidator.validateAccessions( + accessionPreconditionValidator.validateAccessions( username, accessions, listOf(APPROVED_FOR_RELEASE), @@ -455,7 +455,7 @@ class DatabaseService( fun confirmRevocation(accessionVersions: List, username: String, organism: Organism) { log.info { "Confirming revocation for ${accessionVersions.size} sequence entries" } - submissionPreconditionValidator.validateAccessionVersions( + accessionPreconditionValidator.validateAccessionVersions( username, accessionVersions, listOf(AWAITING_APPROVAL_FOR_REVOCATION), @@ -481,7 +481,7 @@ class DatabaseService( fun deleteSequenceEntryVersions(accessionVersions: List, submitter: String, organism: Organism) { log.info { "Deleting accession versions: $accessionVersions" } - submissionPreconditionValidator.validateAccessionVersions( + accessionPreconditionValidator.validateAccessionVersions( submitter, accessionVersions, listOf(RECEIVED, AWAITING_APPROVAL, HAS_ERRORS, AWAITING_APPROVAL_FOR_REVOCATION), @@ -496,7 +496,7 @@ class DatabaseService( fun submitEditedData(submitter: String, editedAccessionVersion: UnprocessedData, organism: Organism) { log.info { "edited sequence entry submitted $editedAccessionVersion" } - submissionPreconditionValidator.validateAccessionVersions( + accessionPreconditionValidator.validateAccessionVersions( submitter, listOf(editedAccessionVersion), listOf(AWAITING_APPROVAL, HAS_ERRORS), @@ -529,7 +529,7 @@ class DatabaseService( "Getting sequence entry ${accessionVersion.displayAccessionVersion()} by $submitter to edit" } - submissionPreconditionValidator.validateAccessionVersions( + accessionPreconditionValidator.validateAccessionVersions( submitter, listOf(accessionVersion), listOf(HAS_ERRORS, AWAITING_APPROVAL), diff --git a/backend/src/main/kotlin/org/loculus/backend/service/submission/UploadDatabaseService.kt b/backend/src/main/kotlin/org/loculus/backend/service/submission/UploadDatabaseService.kt index 4c445dd744..1411c5cb3b 100644 --- a/backend/src/main/kotlin/org/loculus/backend/service/submission/UploadDatabaseService.kt +++ b/backend/src/main/kotlin/org/loculus/backend/service/submission/UploadDatabaseService.kt @@ -44,7 +44,7 @@ private val log = KotlinLogging.logger { } class UploadDatabaseService( private val parseFastaHeader: ParseFastaHeader, private val compressor: CompressionService, - private val submissionPreconditionValidator: SubmissionPreconditionValidator, + private val accessionPreconditionValidator: AccessionPreconditionValidator, private val dataUseTermsDatabaseService: DataUseTermsDatabaseService, ) { @@ -140,8 +140,8 @@ class UploadDatabaseService( val result = if (submissionParams is SubmissionParams.OriginalSubmissionParams) { dataUseTermsDatabaseService.setNewDataUseTerms( - insertionResult.map { it.accession }, submissionParams.username, + insertionResult.map { it.accession }, submissionParams.dataUseTerms, ) @@ -176,7 +176,7 @@ class UploadDatabaseService( .select { uploadIdColumn eq uploadId } .map { it[accessionColumn]!! } - val existingAccessionVersions = submissionPreconditionValidator.validateAccessions( + val existingAccessionVersions = accessionPreconditionValidator.validateAccessions( username, accessions, listOf(Status.APPROVED_FOR_RELEASE), diff --git a/backend/src/main/kotlin/org/loculus/backend/utils/AccessionComparators.kt b/backend/src/main/kotlin/org/loculus/backend/utils/AccessionComparators.kt index 56c39a8068..912c678c8d 100644 --- a/backend/src/main/kotlin/org/loculus/backend/utils/AccessionComparators.kt +++ b/backend/src/main/kotlin/org/loculus/backend/utils/AccessionComparators.kt @@ -7,13 +7,13 @@ typealias Version = Long object AccessionComparator : Comparator { override fun compare(left: Accession, right: Accession): Int { - return left.toInt().compareTo(right.toInt()) + return left.compareTo(right) } } object AccessionVersionComparator : Comparator { override fun compare(left: AccessionVersionInterface, right: AccessionVersionInterface): Int { - return when (val accessionResult = left.accession.toInt().compareTo(right.accession.toInt())) { + return when (val accessionResult = left.accession.compareTo(right.accession)) { 0 -> left.version.compareTo(right.version) else -> accessionResult } diff --git a/backend/src/test/kotlin/org/loculus/backend/service/DataUseTermsTest.kt b/backend/src/test/kotlin/org/loculus/backend/api/DataUseTermsTest.kt similarity index 89% rename from backend/src/test/kotlin/org/loculus/backend/service/DataUseTermsTest.kt rename to backend/src/test/kotlin/org/loculus/backend/api/DataUseTermsTest.kt index fa2bad8df3..af4bbeacf8 100644 --- a/backend/src/test/kotlin/org/loculus/backend/service/DataUseTermsTest.kt +++ b/backend/src/test/kotlin/org/loculus/backend/api/DataUseTermsTest.kt @@ -1,4 +1,4 @@ -package org.loculus.backend.service +package org.loculus.backend.api import com.fasterxml.jackson.databind.ObjectMapper import com.fasterxml.jackson.module.kotlin.readValue @@ -8,7 +8,6 @@ import org.hamcrest.CoreMatchers.`is` import org.hamcrest.MatcherAssert.assertThat import org.junit.jupiter.api.Test import org.loculus.backend.SpringBootTestWithoutDatabase -import org.loculus.backend.api.DataUseTerms import org.springframework.beans.factory.annotation.Autowired @SpringBootTestWithoutDatabase @@ -43,11 +42,11 @@ class DataUseTermsTest(@Autowired private val objectMapper: ObjectMapper) { """.replace("\n", "").replace(" ", ""), ) - assertThat(dataUseTerms, `is`(DataUseTerms.Open())) + assertThat(dataUseTerms, `is`(DataUseTerms.Open)) } @Test - fun `serialized restricted`() { + fun `serialize restricted`() { val restrictedUntil = "2021-02-01" val dataUseTerms = DataUseTerms.Restricted(LocalDate.parse(restrictedUntil)) @@ -66,8 +65,8 @@ class DataUseTermsTest(@Autowired private val objectMapper: ObjectMapper) { } @Test - fun `serialized open`() { - val dataUseTerms = DataUseTerms.Open() + fun `serialize open`() { + val dataUseTerms = DataUseTerms.Open val expected = """ { diff --git a/backend/src/test/kotlin/org/loculus/backend/controller/EndpointTestExtension.kt b/backend/src/test/kotlin/org/loculus/backend/controller/EndpointTestExtension.kt index c5d49f4a6d..a49513879a 100644 --- a/backend/src/test/kotlin/org/loculus/backend/controller/EndpointTestExtension.kt +++ b/backend/src/test/kotlin/org/loculus/backend/controller/EndpointTestExtension.kt @@ -8,10 +8,12 @@ import org.junit.platform.engine.support.descriptor.ClassSource import org.junit.platform.engine.support.descriptor.MethodSource import org.junit.platform.launcher.TestExecutionListener import org.junit.platform.launcher.TestPlan +import org.loculus.backend.controller.datauseterms.DataUseTermsControllerClient import org.loculus.backend.controller.groupmanagement.GroupManagementControllerClient import org.loculus.backend.controller.submission.DEFAULT_USER_NAME import org.loculus.backend.controller.submission.SubmissionControllerClient import org.loculus.backend.controller.submission.SubmissionConvenienceClient +import org.loculus.backend.service.datauseterms.DATA_USE_TERMS_TABLE_NAME import org.loculus.backend.service.groupmanagement.GROUPS_TABLE_NAME import org.loculus.backend.service.groupmanagement.USER_GROUPS_TABLE_NAME import org.loculus.backend.service.submission.METADATA_UPLOAD_TABLE_NAME @@ -36,6 +38,7 @@ import org.testcontainers.containers.PostgreSQLContainer SubmissionControllerClient::class, SubmissionConvenienceClient::class, GroupManagementControllerClient::class, + DataUseTermsControllerClient::class, PublicJwtKeyConfig::class, ) annotation class EndpointTest( @@ -140,7 +143,8 @@ private fun clearDatabaseStatement(): String { "alter sequence $ACCESSION_SEQUENCE_NAME restart with 1; " + "truncate table $USER_GROUPS_TABLE_NAME; " + "truncate $METADATA_UPLOAD_TABLE_NAME; " + - "truncate $SEQUENCE_UPLOAD_TABLE_NAME; \n" + "truncate $SEQUENCE_UPLOAD_TABLE_NAME; " + + "truncate table $DATA_USE_TERMS_TABLE_NAME cascade; \n" } private fun addUsersToGroupStatement(groupName: String, userNames: List): String { diff --git a/backend/src/test/kotlin/org/loculus/backend/controller/datauseterms/DataUseTermsControllerClient.kt b/backend/src/test/kotlin/org/loculus/backend/controller/datauseterms/DataUseTermsControllerClient.kt new file mode 100644 index 0000000000..492876b403 --- /dev/null +++ b/backend/src/test/kotlin/org/loculus/backend/controller/datauseterms/DataUseTermsControllerClient.kt @@ -0,0 +1,30 @@ +package org.loculus.backend.controller.datauseterms + +import com.fasterxml.jackson.databind.ObjectMapper +import org.loculus.backend.api.DataUseTerms +import org.loculus.backend.api.DataUseTermsChangeRequest +import org.loculus.backend.controller.jwtForDefaultUser +import org.loculus.backend.controller.withAuth +import org.springframework.http.MediaType +import org.springframework.test.web.servlet.MockMvc +import org.springframework.test.web.servlet.ResultActions +import org.springframework.test.web.servlet.request.MockMvcRequestBuilders.put + +val DEFAULT_DATA_USE_CHANGE_REQUEST = DataUseTermsChangeRequest( + accessions = listOf("1", "2"), + newDataUseTerms = DataUseTerms.Open, +) + +class DataUseTermsControllerClient(private val mockMvc: MockMvc, private val objectMapper: ObjectMapper) { + fun changeDataUseTerms( + newDataUseTerms: DataUseTermsChangeRequest, + jwt: String? = jwtForDefaultUser, + ): ResultActions { + return mockMvc.perform( + put("/data-use-terms") + .contentType(MediaType.APPLICATION_JSON_VALUE) + .content(objectMapper.writeValueAsString(newDataUseTerms)) + .withAuth(jwt), + ) + } +} diff --git a/backend/src/test/kotlin/org/loculus/backend/controller/datauseterms/DataUseTermsControllerTest.kt b/backend/src/test/kotlin/org/loculus/backend/controller/datauseterms/DataUseTermsControllerTest.kt new file mode 100644 index 0000000000..bee2afbd39 --- /dev/null +++ b/backend/src/test/kotlin/org/loculus/backend/controller/datauseterms/DataUseTermsControllerTest.kt @@ -0,0 +1,140 @@ +package org.loculus.backend.controller.datauseterms + +import kotlinx.datetime.Clock +import kotlinx.datetime.DateTimeUnit.Companion.MONTH +import kotlinx.datetime.TimeZone +import kotlinx.datetime.plus +import kotlinx.datetime.toLocalDateTime +import org.hamcrest.CoreMatchers.containsString +import org.junit.jupiter.api.Test +import org.junit.jupiter.params.ParameterizedTest +import org.junit.jupiter.params.provider.MethodSource +import org.loculus.backend.api.DataUseTerms +import org.loculus.backend.api.DataUseTermsChangeRequest +import org.loculus.backend.controller.EndpointTest +import org.loculus.backend.controller.expectUnauthorizedResponse +import org.loculus.backend.controller.submission.SubmissionConvenienceClient +import org.springframework.beans.factory.annotation.Autowired +import org.springframework.http.MediaType +import org.springframework.test.web.servlet.ResultActions +import org.springframework.test.web.servlet.ResultMatcher +import org.springframework.test.web.servlet.result.MockMvcResultMatchers.content +import org.springframework.test.web.servlet.result.MockMvcResultMatchers.jsonPath +import org.springframework.test.web.servlet.result.MockMvcResultMatchers.status + +private fun dateMonthsFromNow(months: Int) = Clock.System.now().toLocalDateTime(TimeZone.UTC).date.plus(months, MONTH) + +@EndpointTest +class DataUseTermsControllerTest( + @Autowired private val client: DataUseTermsControllerClient, + @Autowired private val submissionConvenienceClient: SubmissionConvenienceClient, +) { + + @ParameterizedTest + @MethodSource("authorizationTestCases") + fun `GIVEN invalid authorization token WHEN performing action THEN returns 401 Unauthorized`( + authScenario: AuthScenario, + ) { + expectUnauthorizedResponse(isModifyingRequest = authScenario.isModifying) { + authScenario.testFunction(it, client) + } + } + + @Test + fun `GIVEN non-existing accessions WHEN setting new data use terms THEN return unprocessable entity`() { + client.changeDataUseTerms(DEFAULT_DATA_USE_CHANGE_REQUEST) + .andExpect(status().isUnprocessableEntity) + .andExpect(content().contentType(MediaType.APPLICATION_JSON_VALUE)) + .andExpect( + jsonPath("\$.detail", containsString("Accessions 1, 2 do not exist")), + ) + } + + @ParameterizedTest + @MethodSource("dataUseTermsTestCases") + fun `test data use terms changes`(testCase: DataUseTermsTestCase) { + submissionConvenienceClient.submitDefaultFiles(dataUseTerms = testCase.setupDataUseTerms) + + val result = client.changeDataUseTerms(testCase.changeRequest) + .andExpect(testCase.expectedStatus) + + if (testCase.expectedContentType != null && testCase.expectedDetailContains != null) { + result + .andExpect(content().contentType(testCase.expectedContentType)) + .andExpect(jsonPath("\$.detail", containsString(testCase.expectedDetailContains))) + } + } + + companion object { + data class AuthScenario( + val testFunction: (String?, DataUseTermsControllerClient) -> ResultActions, + val isModifying: Boolean, + ) + + @JvmStatic + fun authorizationTestCases(): List = listOf( + AuthScenario( + { jwt, client -> + client.changeDataUseTerms( + newDataUseTerms = DEFAULT_DATA_USE_CHANGE_REQUEST, + jwt = jwt, + ) + }, + true, + ), + ) + + data class DataUseTermsTestCase( + val setupDataUseTerms: DataUseTerms, + val changeRequest: DataUseTermsChangeRequest, + val expectedStatus: ResultMatcher, + val expectedContentType: String?, + val expectedDetailContains: String?, + ) + + @JvmStatic + fun dataUseTermsTestCases(): List { + return listOf( + DataUseTermsTestCase( + setupDataUseTerms = DataUseTerms.Open, + changeRequest = DEFAULT_DATA_USE_CHANGE_REQUEST, + expectedStatus = status().isNoContent, + expectedContentType = null, + expectedDetailContains = null, + ), + DataUseTermsTestCase( + setupDataUseTerms = DataUseTerms.Open, + changeRequest = DataUseTermsChangeRequest( + accessions = listOf("1", "2"), + newDataUseTerms = DataUseTerms.Restricted(dateMonthsFromNow(6)), + ), + expectedStatus = status().isUnprocessableEntity, + expectedContentType = MediaType.APPLICATION_JSON_VALUE, + expectedDetailContains = "Cannot change data use terms from OPEN to RESTRICTED.", + ), + DataUseTermsTestCase( + setupDataUseTerms = DataUseTerms.Restricted(dateMonthsFromNow(6)), + changeRequest = DataUseTermsChangeRequest( + accessions = listOf("1", "2"), + newDataUseTerms = DataUseTerms.Restricted(dateMonthsFromNow(-1)), + ), + expectedStatus = status().isBadRequest, + expectedContentType = MediaType.APPLICATION_JSON_VALUE, + expectedDetailContains = "The date 'restrictedUntil' must be in the future, " + + "up to a maximum of 1 year from now.", + ), + DataUseTermsTestCase( + setupDataUseTerms = DataUseTerms.Restricted(dateMonthsFromNow(6)), + changeRequest = DataUseTermsChangeRequest( + accessions = listOf("1", "2"), + newDataUseTerms = DataUseTerms.Restricted(dateMonthsFromNow(7)), + ), + expectedStatus = status().isUnprocessableEntity, + expectedContentType = MediaType.APPLICATION_JSON_VALUE, + expectedDetailContains = "Cannot extend restricted data use period. " + + "Please choose a date before ${dateMonthsFromNow(6)}.", + ), + ) + } + } +} diff --git a/backend/src/test/kotlin/org/loculus/backend/controller/submission/GetDataToEditEndpointTest.kt b/backend/src/test/kotlin/org/loculus/backend/controller/submission/GetDataToEditEndpointTest.kt index f0ac3d9133..563ae38e5f 100644 --- a/backend/src/test/kotlin/org/loculus/backend/controller/submission/GetDataToEditEndpointTest.kt +++ b/backend/src/test/kotlin/org/loculus/backend/controller/submission/GetDataToEditEndpointTest.kt @@ -189,8 +189,11 @@ class GetDataToEditEndpointTest( sequencesToEdit.getAccessionVersions(), containsInAnyOrder(*otherOrganismData.getAccessionVersions().toTypedArray()), ) + + val accessionVersionSet = defaultOrganismData.getAccessionVersions().toSet() + assertThat( - sequencesToEdit.getAccessionVersions().intersect(defaultOrganismData.getAccessionVersions().toSet()), + sequencesToEdit.getAccessionVersions().intersect(accessionVersionSet), `is`(empty()), ) } diff --git a/backend/src/test/kotlin/org/loculus/backend/controller/submission/ReviseEndpointTest.kt b/backend/src/test/kotlin/org/loculus/backend/controller/submission/ReviseEndpointTest.kt index fac884a0cd..38373bc3ab 100644 --- a/backend/src/test/kotlin/org/loculus/backend/controller/submission/ReviseEndpointTest.kt +++ b/backend/src/test/kotlin/org/loculus/backend/controller/submission/ReviseEndpointTest.kt @@ -193,9 +193,8 @@ class ReviseEndpointTest( .andExpect( jsonPath("\$.detail").value( "Accession versions are in not in one of the states [APPROVED_FOR_RELEASE]: " + - "1.1 - HAS_ERRORS, 2.1 - HAS_ERRORS, 3.1 - HAS_ERRORS, 4.1 - HAS_ERRORS, " + - "5.1 - HAS_ERRORS, 6.1 - HAS_ERRORS, 7.1 - HAS_ERRORS, 8.1 - HAS_ERRORS, " + - "9.1 - HAS_ERRORS, 10.1 - HAS_ERRORS", + "1.1 - HAS_ERRORS, 10.1 - HAS_ERRORS, 2.1 - HAS_ERRORS, 3.1 - HAS_ERRORS, 4.1 - HAS_ERRORS, " + + "5.1 - HAS_ERRORS, 6.1 - HAS_ERRORS, 7.1 - HAS_ERRORS, 8.1 - HAS_ERRORS, 9.1 - HAS_ERRORS", ), ) } diff --git a/backend/src/test/kotlin/org/loculus/backend/controller/submission/SubmissionControllerClient.kt b/backend/src/test/kotlin/org/loculus/backend/controller/submission/SubmissionControllerClient.kt index 3194cded3f..5503fa71cb 100644 --- a/backend/src/test/kotlin/org/loculus/backend/controller/submission/SubmissionControllerClient.kt +++ b/backend/src/test/kotlin/org/loculus/backend/controller/submission/SubmissionControllerClient.kt @@ -2,7 +2,7 @@ package org.loculus.backend.controller.submission import com.fasterxml.jackson.databind.ObjectMapper import org.loculus.backend.api.AccessionVersion -import org.loculus.backend.api.DataUseTermsType +import org.loculus.backend.api.DataUseTerms import org.loculus.backend.api.SubmittedProcessedData import org.loculus.backend.api.UnprocessedData import org.loculus.backend.controller.DEFAULT_GROUP_NAME @@ -30,16 +30,21 @@ class SubmissionControllerClient(private val mockMvc: MockMvc, private val objec sequencesFile: MockMultipartFile, organism: String = DEFAULT_ORGANISM, groupName: String = DEFAULT_GROUP_NAME, - dataUseTermType: DataUseTermsType = DataUseTermsType.OPEN, - restrictedUntil: String? = null, + dataUseTerm: DataUseTerms = DataUseTerms.Open, jwt: String? = jwtForDefaultUser, ): ResultActions = mockMvc.perform( multipart(addOrganismToPath("/submit", organism = organism)) .file(sequencesFile) .file(metadataFile) .param("groupName", groupName) - .param("dataUseTermsType", dataUseTermType.name) - .param("restrictedUntil", restrictedUntil) + .param("dataUseTermsType", dataUseTerm.type.name) + .param( + "restrictedUntil", + when (dataUseTerm) { + is DataUseTerms.Restricted -> dataUseTerm.restrictedUntil.toString() + else -> null + }, + ) .withAuth(jwt), ) diff --git a/backend/src/test/kotlin/org/loculus/backend/controller/submission/SubmissionConvenienceClient.kt b/backend/src/test/kotlin/org/loculus/backend/controller/submission/SubmissionConvenienceClient.kt index 99e0ca28bb..7cd9e13722 100644 --- a/backend/src/test/kotlin/org/loculus/backend/controller/submission/SubmissionConvenienceClient.kt +++ b/backend/src/test/kotlin/org/loculus/backend/controller/submission/SubmissionConvenienceClient.kt @@ -4,6 +4,7 @@ import com.fasterxml.jackson.databind.ObjectMapper import com.fasterxml.jackson.module.kotlin.readValue import org.loculus.backend.api.AccessionVersion import org.loculus.backend.api.AccessionVersionInterface +import org.loculus.backend.api.DataUseTerms import org.loculus.backend.api.Organism import org.loculus.backend.api.ProcessedData import org.loculus.backend.api.SequenceEntryStatus @@ -35,6 +36,7 @@ class SubmissionConvenienceClient( username: String = DEFAULT_USER_NAME, groupName: String = DEFAULT_GROUP_NAME, organism: String = DEFAULT_ORGANISM, + dataUseTerms: DataUseTerms = DataUseTerms.Open, ): List { val isMultiSegmented = backendConfig .getInstanceConfig(Organism(organism)) @@ -50,6 +52,7 @@ class SubmissionConvenienceClient( }, organism = organism, groupName = groupName, + dataUseTerm = dataUseTerms, jwt = generateJwtFor(username), ) diff --git a/backend/src/test/kotlin/org/loculus/backend/controller/submission/SubmitEndpointTest.kt b/backend/src/test/kotlin/org/loculus/backend/controller/submission/SubmitEndpointTest.kt index 185963d887..946f6ceb12 100644 --- a/backend/src/test/kotlin/org/loculus/backend/controller/submission/SubmitEndpointTest.kt +++ b/backend/src/test/kotlin/org/loculus/backend/controller/submission/SubmitEndpointTest.kt @@ -12,7 +12,7 @@ import org.junit.jupiter.api.Test import org.junit.jupiter.params.ParameterizedTest import org.junit.jupiter.params.provider.Arguments import org.junit.jupiter.params.provider.MethodSource -import org.loculus.backend.api.DataUseTermsType +import org.loculus.backend.api.DataUseTerms import org.loculus.backend.api.Organism import org.loculus.backend.controller.DEFAULT_GROUP_NAME import org.loculus.backend.controller.DEFAULT_ORGANISM @@ -149,15 +149,13 @@ class SubmitEndpointTest( expectedTitle: String, expectedMessage: String, organism: Organism, - dataUseTermType: DataUseTermsType, - restrictedUntil: String?, + dataUseTerm: DataUseTerms, ) { submissionControllerClient.submit( metadataFile, sequencesFile, organism = organism.name, - dataUseTermType = dataUseTermType, - restrictedUntil = restrictedUntil, + dataUseTerm = dataUseTerm, ) .andExpect(expectedStatus) .andExpect(jsonPath("\$.title").value(expectedTitle)) @@ -207,8 +205,7 @@ class SubmitEndpointTest( "Bad Request", "Required part 'metadataFile' is not present.", DEFAULT_ORGANISM, - DataUseTermsType.OPEN, - null, + DataUseTerms.Open, ), Arguments.of( "sequences file with wrong submitted filename", @@ -218,8 +215,7 @@ class SubmitEndpointTest( "Bad Request", "Required part 'sequenceFile' is not present.", DEFAULT_ORGANISM, - DataUseTermsType.OPEN, - null, + DataUseTerms.Open, ), Arguments.of( "wrong extension for metadata file", @@ -233,8 +229,7 @@ class SubmitEndpointTest( ".${metadataFileTypes.getCompressedExtensions()} " + "for compressed submissions", DEFAULT_ORGANISM, - DataUseTermsType.OPEN, - null, + DataUseTerms.Open, ), Arguments.of( "wrong extension for sequences file", @@ -248,8 +243,7 @@ class SubmitEndpointTest( ".${sequenceFileTypes.getCompressedExtensions()} " + "for compressed submissions", DEFAULT_ORGANISM, - DataUseTermsType.OPEN, - null, + DataUseTerms.Open, ), Arguments.of( "metadata file where one row has a blank header", @@ -265,8 +259,7 @@ class SubmitEndpointTest( "Unprocessable Entity", "A row in metadata file contains no submissionId", DEFAULT_ORGANISM, - DataUseTermsType.OPEN, - null, + DataUseTerms.Open, ), Arguments.of( "metadata file with no header", @@ -281,8 +274,7 @@ class SubmitEndpointTest( "Unprocessable Entity", "The metadata file does not contain the header 'submissionId'", DEFAULT_ORGANISM, - DataUseTermsType.OPEN, - null, + DataUseTerms.Open, ), Arguments.of( "duplicate headers in metadata file", @@ -298,8 +290,7 @@ class SubmitEndpointTest( "Unprocessable Entity", "Metadata file contains at least one duplicate submissionId", DEFAULT_ORGANISM, - DataUseTermsType.OPEN, - null, + DataUseTerms.Open, ), Arguments.of( "duplicate headers in sequence file", @@ -316,8 +307,7 @@ class SubmitEndpointTest( "Unprocessable Entity", "Sequence file contains at least one duplicate submissionId", DEFAULT_ORGANISM, - DataUseTermsType.OPEN, - null, + DataUseTerms.Open, ), Arguments.of( "metadata file misses headers", @@ -339,8 +329,7 @@ class SubmitEndpointTest( "Unprocessable Entity", "Sequence file contains 1 submissionIds that are not present in the metadata file: notInMetadata", DEFAULT_ORGANISM, - DataUseTermsType.OPEN, - null, + DataUseTerms.Open, ), Arguments.of( "sequence file misses headers", @@ -361,8 +350,7 @@ class SubmitEndpointTest( "Unprocessable Entity", "Metadata file contains 1 submissionIds that are not present in the sequence file: notInSequences", DEFAULT_ORGANISM, - DataUseTermsType.OPEN, - null, + DataUseTerms.Open, ), Arguments.of( "FASTA header misses segment name", @@ -383,19 +371,7 @@ class SubmitEndpointTest( "The FASTA header commonHeader does not contain the segment name. Please provide the segment " + "name in the format _", OTHER_ORGANISM, - DataUseTermsType.OPEN, - null, - ), - Arguments.of( - "restricted use data without until date", - DefaultFiles.metadataFile, - DefaultFiles.sequencesFile, - status().isBadRequest, - "Bad Request", - "The date 'restrictedUntil' must be set if 'dataUseTermsType' is RESTRICTED.", - DEFAULT_ORGANISM, - DataUseTermsType.RESTRICTED, - null, + DataUseTerms.Open, ), Arguments.of( "restricted use data with until date in the past", @@ -405,8 +381,8 @@ class SubmitEndpointTest( "Bad Request", "The date 'restrictedUntil' must be in the future, up to a maximum of 1 year from now.", DEFAULT_ORGANISM, - DataUseTermsType.RESTRICTED, - now.minus(1, DAY).toString(), + DataUseTerms.Restricted(now.minus(1, DAY)), + ), Arguments.of( "restricted use data with until date further than 1 year", @@ -416,8 +392,7 @@ class SubmitEndpointTest( "Bad Request", "The date 'restrictedUntil' must not exceed 1 year from today.", DEFAULT_ORGANISM, - DataUseTermsType.RESTRICTED, - now.plus(2, YEAR).toString(), + DataUseTerms.Restricted(now.plus(2, YEAR)), ), ) }