feat(website, backend): extend approve/delete with scope

 * adding scope to `get-sequences` `approve` and `delete` endpoint
 * remove browser-testrunner from testGroup and giving them all separate groups

backend/src/main/kotlin/org/loculus/backend/api/SubmissionTypes.kt

@@ -35,6 +35,61 @@ data class SubmissionIdMapping(
 
 fun <T : AccessionVersionInterface> List<T>.toPairs() = map { Pair(it.accession, it.version) }
 
+data class AccessionVersions(
+    val accessionVersions: List<AccessionVersion>,
+)
+
+@Schema(
+    description = "If set to 'INCLUDE_WARNINGS', sequence entries with warnings are included in the response." +
+        " If set to 'EXCLUDE_WARNINGS', sequence entries with warnings are not included in the response. " +
+        "Default is 'INCLUDE_WARNINGS'.",
+)
+enum class WarningsFilter {
+    EXCLUDE_WARNINGS,
+    INCLUDE_WARNINGS,
+}
+
+enum class DeleteSequenceScope {
+    ALL,
+    PROCESSED_WITH_ERRORS,
+    PROCESSED_WITH_WARNINGS,
+}
+
+const val ACCESSION_VERSIONS_FILTER_DESCRIPTION =
+    "A List of accession versions that the operation will be restricted to. " +
+        "Omit or set to null to consider all sequences."
+
+data class AccessionVersionsFilterWithDeletionScope(
+    @Schema(
+        description = ACCESSION_VERSIONS_FILTER_DESCRIPTION,
+    )
+    val accessionVersionsFilter: List<AccessionVersion>? = null,
+    @Schema(
+        description = "Scope for deletion. If scope is set to 'ALL', all sequences are deleted. " +
+            "If scope is set to 'PROCESSED_WITH_ERRORS', only processed sequences with errors are deleted. " +
+            "If scope is set to 'PROCESSED_WITH_WARNINGS', only processed sequences in `AWAITING_APPROVAL` " +
+            "with warnings are deleted.",
+    )
+    val scope: DeleteSequenceScope,
+)
+
+enum class ApproveDataScope {
+    ALL,
+    WITHOUT_WARNINGS,
+}
+
+data class AccessionVersionsFilterWithApprovalScope(
+    @Schema(
+        description = ACCESSION_VERSIONS_FILTER_DESCRIPTION,
+    )
+    val accessionVersionsFilter: List<AccessionVersion>? = null,
+    @Schema(
+        description = "Scope for approval. If scope is set to 'ALL', all sequences are approved. " +
+            "If scope is set to 'WITHOUT_WARNINGS', only sequences without warnings are approved.",
+    )
+    val scope: ApproveDataScope,
+)
+
 data class SubmittedProcessedData(
     override val accession: Accession,
     override val version: Version,

backend/src/main/kotlin/org/loculus/backend/controller/SubmissionController.kt

@@ -11,6 +11,9 @@ import jakarta.validation.Valid
 import jakarta.validation.constraints.Max
 import mu.KotlinLogging
 import org.loculus.backend.api.AccessionVersion
+import org.loculus.backend.api.AccessionVersions
+import org.loculus.backend.api.AccessionVersionsFilterWithApprovalScope
+import org.loculus.backend.api.AccessionVersionsFilterWithDeletionScope
 import org.loculus.backend.api.Accessions
 import org.loculus.backend.api.DataUseTerms
 import org.loculus.backend.api.DataUseTermsType
@@ -22,6 +25,7 @@ import org.loculus.backend.api.Status
 import org.loculus.backend.api.SubmissionIdMapping
 import org.loculus.backend.api.SubmittedProcessedData
 import org.loculus.backend.api.UnprocessedData
+import org.loculus.backend.api.WarningsFilter
 import org.loculus.backend.model.ReleasedDataModel
 import org.loculus.backend.model.SubmissionParams
 import org.loculus.backend.model.SubmitModel
@@ -252,6 +256,8 @@ class SubmissionController(
         @RequestParam(required = false)
         statusesFilter: List<Status>?,
         @UsernameFromJwt username: String,
+        @RequestParam(required = false, defaultValue = "INCLUDE_WARNINGS")
+        warningsFilter: WarningsFilter,
         @Parameter(
             description = "Part of pagination parameters. Page number starts from 0. " +
                 "If page or size are not provided, all sequences are returned.",
@@ -264,8 +270,15 @@ class SubmissionController(
         )
         @RequestParam(required = false)
         size: Int?,
-    ): GetSequenceResponse =
-        submissionDatabaseService.getSequences(username, organism, groupsFilter, statusesFilter, page, size)
+    ): GetSequenceResponse = submissionDatabaseService.getSequences(
+        username,
+        organism,
+        groupsFilter,
+        statusesFilter,
+        warningsFilter,
+        page,
+        size,
+    )
 
     @Operation(description = APPROVE_PROCESSED_DATA_DESCRIPTION)
     @ResponseStatus(HttpStatus.NO_CONTENT)
@@ -274,9 +287,15 @@ class SubmissionController(
         @PathVariable @Valid
         organism: Organism,
         @UsernameFromJwt username: String,
-        @RequestBody body: AccessionVersions,
+        @RequestBody
+        body: AccessionVersionsFilterWithApprovalScope,
     ) {
-        submissionDatabaseService.approveProcessedData(username, body.accessionVersions, organism)
+        submissionDatabaseService.approveProcessedData(
+            submitter = username,
+            accessionVersionsFilter = body.accessionVersionsFilter,
+            organism = organism,
+            scope = body.scope,
+        )
     }
 
     @Operation(description = REVOKE_DESCRIPTION)
@@ -299,16 +318,22 @@ class SubmissionController(
     ) = submissionDatabaseService.confirmRevocation(body.accessionVersions, username, organism)
 
     @Operation(description = DELETE_SEQUENCES_DESCRIPTION)
-    @ResponseStatus(HttpStatus.NO_CONTENT)
+    @ResponseStatus(HttpStatus.OK)
     @DeleteMapping(
         "/delete-sequence-entry-versions",
     )
     fun deleteSequence(
         @PathVariable @Valid
         organism: Organism,
         @UsernameFromJwt username: String,
-        @RequestBody body: AccessionVersions,
-    ) = submissionDatabaseService.deleteSequenceEntryVersions(body.accessionVersions, username, organism)
+        @RequestBody
+        body: AccessionVersionsFilterWithDeletionScope,
+    ): List<AccessionVersion> = submissionDatabaseService.deleteSequenceEntryVersions(
+        body.accessionVersionsFilter,
+        username,
+        organism,
+        body.scope,
+    )
 
     private fun <T> stream(sequenceProvider: () -> Sequence<T>) = StreamingResponseBody { outputStream ->
         try {
@@ -320,10 +345,6 @@ class SubmissionController(
             )
         }
     }
-
-    data class AccessionVersions(
-        val accessionVersions: List<AccessionVersion>,
-    )
 }
 
 @Target(AnnotationTarget.VALUE_PARAMETER)

backend/src/main/kotlin/org/loculus/backend/service/submission/SequenceEntriesTable.kt

@@ -9,6 +9,7 @@ import org.jetbrains.exposed.sql.SqlExpressionBuilder.inList
 import org.jetbrains.exposed.sql.Table
 import org.jetbrains.exposed.sql.alias
 import org.jetbrains.exposed.sql.and
+import org.jetbrains.exposed.sql.json.exists
 import org.jetbrains.exposed.sql.json.jsonb
 import org.jetbrains.exposed.sql.kotlin.datetime.datetime
 import org.jetbrains.exposed.sql.max
@@ -81,8 +82,6 @@ class SequenceEntriesDataTable(
         )
     }
 
-    val isMaxReleasedVersion = versionColumn eq maxReleasedVersionQuery()
-
     private fun maxReleasedVersionQuery(): Expression<Long?> {
         val subQueryTable = alias("subQueryTable")
         return wrapAsExpression(
@@ -100,6 +99,8 @@ class SequenceEntriesDataTable(
 
     fun organismIs(organism: Organism) = organismColumn eq organism.name
 
+    val entriesWithWarnings = warningsColumn.exists("[0]")
+
     fun statusIs(status: Status) = statusColumn eq status.name
 
     fun statusIsOneOf(statuses: List<Status>) = statusColumn inList statuses.map { it.name }

backend/src/main/kotlin/org/loculus/backend/service/submission/SubmissionDatabaseService.kt

@@ -11,6 +11,7 @@ import kotlinx.datetime.toLocalDateTime
 import mu.KotlinLogging
 import org.jetbrains.exposed.sql.Database
 import org.jetbrains.exposed.sql.JoinType
+import org.jetbrains.exposed.sql.Op
 import org.jetbrains.exposed.sql.SqlExpressionBuilder.plus
 import org.jetbrains.exposed.sql.and
 import org.jetbrains.exposed.sql.andWhere
@@ -19,13 +20,16 @@ import org.jetbrains.exposed.sql.deleteWhere
 import org.jetbrains.exposed.sql.insert
 import org.jetbrains.exposed.sql.kotlin.datetime.dateTimeParam
 import org.jetbrains.exposed.sql.max
+import org.jetbrains.exposed.sql.not
 import org.jetbrains.exposed.sql.select
 import org.jetbrains.exposed.sql.stringParam
 import org.jetbrains.exposed.sql.update
 import org.loculus.backend.api.AccessionVersion
 import org.loculus.backend.api.AccessionVersionInterface
+import org.loculus.backend.api.ApproveDataScope
 import org.loculus.backend.api.DataUseTerms
 import org.loculus.backend.api.DataUseTermsType
+import org.loculus.backend.api.DeleteSequenceScope
 import org.loculus.backend.api.GetSequenceResponse
 import org.loculus.backend.api.Group
 import org.loculus.backend.api.Organism
@@ -42,6 +46,7 @@ import org.loculus.backend.api.Status.RECEIVED
 import org.loculus.backend.api.SubmissionIdMapping
 import org.loculus.backend.api.SubmittedProcessedData
 import org.loculus.backend.api.UnprocessedData
+import org.loculus.backend.api.WarningsFilter
 import org.loculus.backend.controller.BadRequestException
 import org.loculus.backend.controller.ProcessingValidationException
 import org.loculus.backend.controller.UnprocessableEntityException
@@ -217,26 +222,58 @@ class SubmissionDatabaseService(
         }
     }
 
-    fun approveProcessedData(submitter: String, accessionVersions: List<AccessionVersion>, organism: Organism) {
-        log.info { "approving ${accessionVersions.size} sequences by $submitter" }
+    fun approveProcessedData(
+        submitter: String,
+        accessionVersionsFilter: List<AccessionVersion>?,
+        organism: Organism,
+        scope: ApproveDataScope,
+    ) {
+        if (accessionVersionsFilter == null) {
+            log.info { "approving all sequences by all groups $submitter is member of" }
+        } else {
+            log.info { "approving ${accessionVersionsFilter.size} sequences by $submitter" }
+        }
 
         val now = Clock.System.now().toLocalDateTime(TimeZone.UTC)
 
-        accessionPreconditionValidator.validateAccessionVersions(
-            submitter,
-            accessionVersions,
-            listOf(AWAITING_APPROVAL),
-            organism,
-        )
+        if (accessionVersionsFilter != null) {
+            accessionPreconditionValidator.validateAccessionVersions(
+                submitter,
+                accessionVersionsFilter,
+                listOf(AWAITING_APPROVAL),
+                organism,
+            )
+        }
 
         sequenceEntriesTableProvider.get(organism).let { table ->
-            table.update(
+            table.join(
+                DataUseTermsTable,
+                JoinType.LEFT,
+                additionalConstraint = {
+                    (table.accessionColumn eq DataUseTermsTable.accessionColumn) and
+                        (DataUseTermsTable.isNewestDataUseTerms)
+                },
+            ).update(
                 where = {
-                    table.accessionVersionIsIn(accessionVersions) and table.statusIs(AWAITING_APPROVAL)
+                    val statusCondition = table.statusIs(AWAITING_APPROVAL)
+
+                    val accessionCondition = if (accessionVersionsFilter !== null) {
+                        table.accessionVersionIsIn(accessionVersionsFilter)
+                    } else {
+                        table.groupIsOneOf(groupManagementDatabaseService.getGroupsOfUser(submitter))
+                    }
+
+                    val scopeCondition = if (scope == ApproveDataScope.WITHOUT_WARNINGS) {
+                        not(table.entriesWithWarnings)
+                    } else {
+                        Op.TRUE
+                    }
+
+                    statusCondition and accessionCondition and scopeCondition
                 },
             ) {
-                it[statusColumn] = APPROVED_FOR_RELEASE.name
-                it[releasedAtColumn] = now
+                it[table.statusColumn] = APPROVED_FOR_RELEASE.name
+                it[table.releasedAtColumn] = now
             }
         }
     }
@@ -364,6 +401,7 @@ class SubmissionDatabaseService(
         organism: Organism?,
         groupsFilter: List<String>?,
         statusesFilter: List<Status>?,
+        warningsFilter: WarningsFilter? = null,
         page: Int? = null,
         size: Int? = null,
     ): GetSequenceResponse {
@@ -419,6 +457,12 @@ class SubmissionDatabaseService(
                 query.count { it[table.statusColumn] == status.name }
             }
 
+            if (warningsFilter == WarningsFilter.EXCLUDE_WARNINGS) {
+                query.andWhere {
+                    not(table.entriesWithWarnings)
+                }
+            }
+
             val pagedQuery = if (page != null && size != null) {
                 query.limit(size, (page * size).toLong())
             } else {
@@ -540,19 +584,66 @@ class SubmissionDatabaseService(
         }
     }
 
-    fun deleteSequenceEntryVersions(accessionVersions: List<AccessionVersion>, submitter: String, organism: Organism) {
-        log.info { "Deleting accession versions: $accessionVersions" }
+    fun deleteSequenceEntryVersions(
+        accessionVersionsFilter: List<AccessionVersion>?,
+        submitter: String,
+        organism: Organism,
+        scope: DeleteSequenceScope,
+    ): List<AccessionVersion> {
+        if (accessionVersionsFilter == null) {
+            log.info { "deleting all sequences of all groups $submitter is member of in the scope $scope" }
+        } else {
+            log.info { "deleting ${accessionVersionsFilter.size} sequences by $submitter in scope $scope" }
+        }
 
-        accessionPreconditionValidator.validateAccessionVersions(
-            submitter,
-            accessionVersions,
-            listOf(RECEIVED, AWAITING_APPROVAL, HAS_ERRORS, AWAITING_APPROVAL_FOR_REVOCATION),
-            organism,
-        )
+        val listOfDeletableStatuses = listOf(RECEIVED, AWAITING_APPROVAL, HAS_ERRORS, AWAITING_APPROVAL_FOR_REVOCATION)
+
+        if (accessionVersionsFilter != null) {
+            accessionPreconditionValidator.validateAccessionVersions(
+                submitter,
+                accessionVersionsFilter,
+                listOfDeletableStatuses,
+                organism,
+            )
+        }
+
+        val sequenceEntriesToDelete = sequenceEntriesTableProvider.get(organism).let { table ->
+            table.join(
+                DataUseTermsTable,
+                JoinType.LEFT,
+                additionalConstraint = {
+                    (table.accessionColumn eq DataUseTermsTable.accessionColumn) and
+                        (DataUseTermsTable.isNewestDataUseTerms)
+                },
+            ).select {
+                val accessionCondition = if (accessionVersionsFilter != null) {
+                    table.accessionVersionIsIn(accessionVersionsFilter)
+                } else {
+                    table.groupIsOneOf(groupManagementDatabaseService.getGroupsOfUser(submitter))
+                }
+
+                val scopeCondition = when (scope) {
+                    DeleteSequenceScope.PROCESSED_WITH_ERRORS -> {
+                        table.statusIs(HAS_ERRORS)
+                    }
+                    DeleteSequenceScope.PROCESSED_WITH_WARNINGS -> {
+                        table.statusIs(AWAITING_APPROVAL) and
+                            table.entriesWithWarnings
+                    }
+                    DeleteSequenceScope.ALL -> table.statusIsOneOf(listOfDeletableStatuses)
+                }
+
+                accessionCondition and scopeCondition
+            }.map {
+                AccessionVersion(it[table.accessionColumn], it[table.versionColumn])
+            }
+        }
 
         sequenceEntriesTableProvider.get(organism).deleteWhere {
-            accessionVersionIsIn(accessionVersions)
+            accessionVersionIsIn(sequenceEntriesToDelete)
         }
+
+        return sequenceEntriesToDelete
     }
 
     fun submitEditedData(submitter: String, editedAccessionVersion: UnprocessedData, organism: Organism) {